Allow running with incomplete descriptions

.circleci/config.yml

@@ -1,19 +1,54 @@
 # Python CircleCI 2.1 configuration file
 # for mets-mods2tei
 #
-# Check https://circleci.com/docs/2.1/language-python/ for more details
+# Check https://circleci.com/docs/2.0/language-python/ for more details
 #
 version: 2.1
 orbs:
   codecov: codecov/codecov@1.0.5
 jobs:
-  build:
+  test:
+    parameters:
+      version:
+        type: string
     docker:
-      - image: python:3.6
+      - image: circleci/python:<< parameters.version >>
     working_directory: ~/repo
     steps:
       - checkout
-      - run: pip install -r requirements-test.txt
-      - run: pip install .
+      - run: make deps deps-test
+      - run: make install
+      - run: make test
       - run: make coverage
       - codecov/upload
+  pypi:
+    docker:
+      - image: circleci/python:3.6
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run: make install
+      - run: python setup.py sdist
+      - run: |
+          pip install cibuildwheel
+          cibuildwheel --output-dir dist
+      - store_artifacts:
+          path: dist/
+          destination: artifacts
+    # later: upload to PyPI...
+
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              version: [3.6.15, 3.7.12, 3.8.12, 3.9.9]
+  deploy:
+    jobs:
+      - pypi:
+          filters:
+            branches:
+              only: master

CHANGELOG.md

@@ -0,0 +1,64 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+### Added
+- tests for TEI API
+- tests for insertion index identification
+- more logging
+- CLI param for output file
+- CLI param for image fileGrp
+
+### Changed
+- Add `front`, `body` and `back` per default
+- Log to stderr instead of stdout
+- Differentiate between (physical) image nr and (logical) page nr
+
+### Fixed
+- Evaluate texts from all struct types but `binding` and `colour_checker`, #43
+- Handle errors during language code expansion, and fallback to `Unbekannt`, #47
+- Add ALTO `HYP` text content if available, #52
+- Allow empty logical structMap and structLink, fallback to physical, or empty, #57
+- Allow partial dmdSec (MODS) or amdSec, fallback to empty, #46, #51
+- Pass all `mods:identifier`s to `msIdentifier/idno` (not just VD and URN)
+- Parse full `titleInfo` (main/sub/part/volume), and re-use in `biblFull`
+- Prefer `titleInfo/title` over `div/@LABEL` if available
+- Map top logical `div/@TYPE` into allowed `biblFull/title/@level` only
+- Map top logical `div/@TYPE` into appropriate `bibl/@type` if possible
+
+## [0.1.1] - 2020-05-11
+### Added
+- Make full text file group selectable by user
+- Add poor man's namespace versioning handling
+
+### Changed
+- Make extraction of subtitles conditional on their presence
+- Use "licence" for all types of licences (even unknown ones), #39
+
+### Fixed
+- Handle nested `@ADMID="AMD"` divs in logical `structMap` (i.e. newspaper case), #43
+- Allow for local path entries (in addition to URLs) in METS, #41
+- Add special treatment for URNs and VD IDs, #37
+
+## [0.1.0] - 2019-12-04
+### Added
+- Correctly place structures which are not on top of a page
+- Set `corresp` and `facs` attributes of `pb` elements
+- Store links to `DEFAULT` images in METS
+- Tests for new functionality
+- Add Changelog file, #28
+
+### Changed
+- Retrieve ALTO files via a dedicated struct link member of the class `Mets`
+- Move text retrieval to `Alto` class
+
+### Removed
+- Get rid of code artifacts carried over from `tocrify`
+
+<!-- link-labels -->
+[unreleased]: ../../compare/v0.1.1...master
+[0.1.1]: ../../compare/v0.1.0...v0.1.1
+[0.1.0]: ../../compare/v1.0...v0.1.0

Makefile

@@ -1,26 +1,40 @@
 # Python interpreter. Default: '$(PYTHON)'
-PYTHON = python
+PYTHON ?= python
+PIP ?= pip
 
 # BEGIN-EVAL makefile-parser --make-help Makefile
 
 help:
 	@echo ""
 	@echo "  Targets"
 	@echo ""
+	@echo "    install   Install this package"
+	@echo "    deps      Install dependencies only"
+	@echo "    deps-test Install dependencies for testing only"
 	@echo "    test      Run all unit tests"
 	@echo "    coverage  Run coverage tests"
 	@echo ""
 	@echo "  Variables"
 	@echo ""
 	@echo "    PYTHON  Python interpreter. Default: '$(PYTHON)'"
+	@echo "    PIP     Python packager. Default: '$(PIP)'"
 
 # END-EVAL
 
 #
 # Tests
 #
 
-.PHONY: test coverage
+.PHONY: install test coverage deps deps-test
+
+install:
+	$(PIP) install .
+
+deps:
+	$(PIP) install -r requirements.txt
+
+deps-test:
+	$(PIP) install -r requirements-test.txt
 
 # Run all unit tests
 test:

README.md

@@ -106,17 +106,27 @@ Usage: mm2tei [OPTIONS] METS
 
   METS: File containing or URL pointing to the METS/MODS XML to be converted
 
+  Parse given METS and its meta-data, and convert it to TEI.
+
+  If `--ocr` is given, then also read the ALTO full-text files from the
+  fileGrp in `--text-group`, and convert page contents accordingly (in
+  physical order). Decorate page boundaries with image and page numbers, and
+  reference the corresponding base image files from `--img-group`.
+
+  Output XML to `--output (use '-' for stdout), log to stderr.`
+
 Options:
+  -O, --output FILENAME           File path to write TEI output to
   -o, --ocr                       Serialize OCR into resulting TEI
-  -T, --text-group TEXT           File group which contains the full text
+  -T, --text-group TEXT           File group which contains the full-text
+  -I, --img-group TEXT            File group which contains the images
   -l, --log-level [DEBUG|INFO|WARN|ERROR|OFF]
-  --help                          Show this message and exit.
+  -h, --help                      Show this message and exit.
 ```
 
 It reads METS XML via URL or file argument and prints the resulting TEI,
 including the extracted information from the MODS part of the METS.
 
 Example:
 
-    mm2tei "https://digital.slub-dresden.de/oai/?verb=GetRecord&metadataPrefix=mets&identifier=oai:de:slub-dresden:db:id-453779263"
-
+    mm2tei -O tei.xml "https://digital.slub-dresden.de/oai/?verb=GetRecord&metadataPrefix=mets&identifier=oai:de:slub-dresden:db:id-453779263"

mets_mods2tei/api/alto.py

@@ -92,7 +92,11 @@ def get_text_in_line(self, line):
         Returns the ALTO-encoded text .
         :param Element line: The line to extract the text from.
         """
-        return " ".join(element.get("CONTENT") for element in line.xpath("./alto:String", namespaces=ns))
+        text = " ".join(element.get("CONTENT") for element in line.xpath("./alto:String", namespaces=ns))
+        hyp = line.find("alto:HYP", namespaces=ns)
+        if hyp is not None:
+            text += hyp.get("CONTENT")
+        return text
 
     def __compute_fuzzy_distance(self, text1, text2):
         """