Add XSD validators for PAGE and METS, #449

.envrc

@@ -0,0 +1 @@
+export PYLINTRC=$PWD/.pylintrc

.vimrc

@@ -0,0 +1,5 @@
+set path+=ocrd
+set path+=ocrd_models
+set path+=ocrd_modelfactory
+set path+=ocrd_validators
+set path+=ocrd_utils

ocrd_validators/ocrd_validators/__init__.py

@@ -7,7 +7,10 @@
     'PageValidator',
     'OcrdToolValidator',
     'OcrdZipValidator',
-    'ValidationReport'
+    'ValidationReport',
+    'XsdValidator',
+    'XsdMetsValidator',
+    'XsdPageValidator',
 ]
 
 from .report import ValidationReport
@@ -16,3 +19,6 @@
 from .page_validator import PageValidator
 from .ocrd_tool_validator import OcrdToolValidator
 from .ocrd_zip_validator import OcrdZipValidator
+from .xsd_validator import XsdValidator
+from .xsd_mets_validator import XsdMetsValidator
+from .xsd_page_validator import XsdPageValidator

ocrd_validators/ocrd_validators/constants.py

@@ -2,7 +2,7 @@
 Constants for ocrd_validators.
 """
 import yaml
-from pkg_resources import resource_string
+from pkg_resources import resource_string, resource_filename
 
 OCRD_TOOL_SCHEMA = yaml.safe_load(resource_string(__name__, 'ocrd_tool.schema.yml'))
 OCRD_BAGIT_PROFILE = yaml.safe_load(resource_string(__name__, 'bagit-profile.yml'))
@@ -12,3 +12,8 @@
 FILE_GROUP_CATEGORIES = ['IMG', 'SEG', 'OCR', 'COR', 'GT']
 TMP_BAGIT_PREFIX = 'ocrd-bagit-'
 OCRD_BAGIT_PROFILE_URL = 'https://ocr-d.github.io/bagit-profile.json'
+XSD_METS_URL = 'https://www.loc.gov/standards/mets/mets.xsd'
+XSD_PAGE_URL = 'http://www.primaresearch.org/schema/PAGE/gts/pagecontent/2019-07-15/pagecontent.xsd'
+XSD_PATHS = {}
+XSD_PATHS[XSD_METS_URL] = resource_filename(__name__, 'xsd/mets.xsd')
+XSD_PATHS[XSD_PAGE_URL] = resource_filename(__name__, 'xsd/page.xsd')

ocrd_validators/ocrd_validators/workspace_validator.py

@@ -12,6 +12,8 @@
 from .constants import FILE_GROUP_CATEGORIES, FILE_GROUP_PREFIX
 from .report import ValidationReport
 from .page_validator import PageValidator
+from .xsd_page_validator import XsdPageValidator
+from .xsd_mets_validator import XsdMetsValidator
 
 log = getLogger('ocrd.workspace_validator')
 
@@ -123,7 +125,11 @@ def _validate(self):
                     self._validate_imagefilename()
                 if 'page' not in self.skip:
                     self._validate_page()
-            except Exception:
+                if 'page_xsd' not in self.skip:
+                    self._validate_page_xsd()
+                if 'mets_xsd' not in self.skip:
+                    self._validate_mets_xsd()
+            except Exception: # pylint: disable=broad-except
                 self.report.add_error("Validation aborted with exception: %s" % format_exc())
         return self.report
 
@@ -261,3 +267,23 @@ def _validate_page(self):
                                                  check_coords=self.page_coordinate_consistency in ['poly', 'both'],
                                                  check_baseline=self.page_coordinate_consistency in ['baseline', 'both'])
             self.report.merge_report(page_report)
+
+    def _validate_page_xsd(self):
+        """
+        Validate all PAGE-XML files against PAGE XSD schema
+        """
+        log.info("Validating all PAGE-XML files against XSD")
+        for ocrd_file in self.mets.find_files(mimetype=MIMETYPE_PAGE, local_only=True):
+            self.workspace.download_file(ocrd_file)
+            for err in XsdPageValidator.validate(Path(ocrd_file.local_filename)).errors:
+                self.report.add_error("%s: %s" % (ocrd_file.ID, err))
+        log.info("Finished alidating all PAGE-XML files against XSD")
+
+    def _validate_mets_xsd(self):
+        """
+        Validate METS against METS XSD schema
+        """
+        log.info("Validating METS %s against XSD" % self.workspace.mets_target)
+        for err in XsdMetsValidator.validate(Path(self.workspace.mets_target)).errors:
+            self.report.add_error("%s: %s" % (self.workspace.mets_target, err))
+        log.info("Finished Validating METS against XSD")