Merge branch 'dev' into 'master'

Merge release version 1.2.0 into master

See merge request zzhd/scf!70

CHANGELOG.txt

@@ -1,3 +1,18 @@
+-----------------------------
+Version 1.2.0
+-----------------------------
+
+STL2STLXML
+- ADDED #44: Options to discard UDA and/or User Data TTI blocks
+
+STLXML-SplitBlocks
+- ADDED #52: Module STLXML-SplitBlocks
+
+STLXML2STL
+- ADDED: Code page 437 support
+- FIXED #51: Missing special characters in CCT 00
+- FIXED: Missing house character in code page 850
+
 -----------------------------
 Version 1.1.1
 -----------------------------

README.md

@@ -4,7 +4,7 @@ converting XML based subtitle formats. Main target is to build up a
 flexible and extensible transformation pipeline to convert EBU STL
 formats and EBU-TT subtitle formats.
 
-The SCF is a stable release with the version 1.1.1.
+The SCF is a stable release with the version 1.2.0.
 
 ## License
 The Subtitle Conversion Framework ("SCF") offered by Institut fuer
@@ -15,8 +15,8 @@ For the XSLT modules an XSLT processor that is conformant to XSLT 1.0 is
 needed. You could use for example a [Saxon XSLT processor](http://saxon.sourceforge.net/)
 from version 6.5.5.
 
-The STLXML2EBU-TT and EBU-TT2STLXML modules require XSLT 2.0 support
-(or instead EXSLT support).
+The STLXML2EBU-TT, EBU-TT2STLXML and STLXML-SplitBlocks modules require XSLT 2.0 support. For
+the STLXML2EBU-TT and EBU-TT2STLXML modules EXSLT support can be used as fallback.
 
 To validate an STLXML document with the STLXML W3C XML Schema an
 XML Schema 1.0 parser is required. You could use for example
@@ -54,10 +54,11 @@ output).
 ## DESCRIPTION
 
 ### Modules
-Currently the SCF has the following seven core modules:
+Currently the SCF has the following eight core modules:
 
 * STLXML-XSD
 * STL2STLXML
+* STLXML-SplitBlocks
 * STLXML2STL
 * STLXML2EBU-TT
 * EBU-TT2STLXML
@@ -79,6 +80,11 @@ The STL2STLXML script decodes the EBU STL file and exports it in an XML
 representation that can be used for further processing with XML
 technologies or for debugging purposes.
 
+#### STLXML-SplitBlocks
+The STLXML-SplitBlocks module splits TTI blocks that exceed the maximum
+Text Field (TF) size in EBU STL. It should be applied to an STLXML file
+before processing that file with STLXML2STL.
+
 #### STLXML2STL
 The STLXML2STL module converts an XML representation of EBU STL to a
 binary file that conforms to EBU STL.

misc/ci/.gitlab-ci.yml

@@ -0,0 +1,218 @@
+image: $scf_ci_image
+
+before_script:
+  # add deployment keys
+  - echo "$id_rsa_deploy_ro" >  ~/.ssh/id_rsa_deploy_ro
+  - chmod 600 ~/.ssh/id_rsa_deploy_ro
+  # Get scf_test_driver repo (last commit only)
+  - git clone --depth 1 $scf_ci_repo_test_driver scf_test_driver
+
+after_script:
+  # remove deployment keys
+  - rm ~/.ssh/id_rsa_deploy_ro
+
+
+# Test pipeline 1: STL -> STLXML -> EBU-TT -> EBU-TT-D -> EBU-TT-D-Basic-DE
+test_pipeline_1:
+  tags:
+    - docker
+  stage: test
+  script:
+    # STL -> STLXML
+    - modules/STL2STLXML/stl2stlxml.py -p -x 2_STLXML.xml misc/ci/test_pipeline_1/1_STL.stl
+    - diff misc/ci/test_pipeline_1/2_STLXML.xml 2_STLXML.xml
+    # STLXML -> EBU-TT (result is patched to be reproducible)
+    - java -cp ~/tools/SaxonEE9/saxon9ee.jar net.sf.saxon.Transform \!indent=true -s:2_STLXML.xml -xsl:modules/STLXML2EBU-TT/STLXML2EBU-TT.xslt -o:3_EBU-TT_tmp.xml
+    - java -cp ~/tools/SaxonEE9/saxon9ee.jar net.sf.saxon.Transform \!indent=true -s:3_EBU-TT_tmp.xml -xsl:misc/ci/make_reproducible_EBU-TT.xsl -o:3_EBU-TT.xml
+    - diff misc/ci/test_pipeline_1/3_EBU-TT.xml 3_EBU-TT.xml
+    # EBU-TT -> EBU-TT-D
+    - java -cp ~/tools/SaxonEE9/saxon9ee.jar net.sf.saxon.Transform \!indent=true -s:3_EBU-TT.xml -xsl:modules/EBU-TT2EBU-TT-D/EBU-TT2EBU-TT-D.xslt -o:4_EBU-TT-D.xml
+    - diff misc/ci/test_pipeline_1/4_EBU-TT-D.xml 4_EBU-TT-D.xml
+    # EBU-TT-D -> EBU-TT-D-Basic-DE
+    - java -cp ~/tools/SaxonEE9/saxon9ee.jar net.sf.saxon.Transform \!indent=true -s:4_EBU-TT-D.xml -xsl:modules/EBU-TT-D2EBU-TT-D-Basic-DE/EBU-TT-D2EBU-TT-D-Basic-DE.xslt -o:5_EBU-TT-D-Basic-DE.xml
+    - diff misc/ci/test_pipeline_1/5_EBU-TT-D-Basic-DE.xml 5_EBU-TT-D-Basic-DE.xml
+
+tests_STL2STLXML:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_stl2stlxml.sh
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_STLXML2STL:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_stlxml2stl.sh
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_STLXML2EBU-TT:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh STLXML2EBU-TT STLXML2EBU-TT STLXML2EBU-TT.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_STLXML-SplitBlocks:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh STLXML-SplitBlocks STLXML-SplitBlocks STLXML-SplitBlocks.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_EBU-TT2STLXML:
+  # to be started by hand, as module has beta status!
+  when: manual
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh EBU-TT2STLXML EBU-TT2STLXML EBU-TT2STLXML.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_EBU-TT2EBU-TT-D:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh EBU-TT2EBU-TT-D EBU-TT2EBU-TT-D EBU-TT2EBU-TT-D.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_EBU-TT-D2EBU-TT-D-Basic-DE:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh EBU-TT-D2EBU-TT-D-Basic-DE EBU-TT-D2EBU-TT-D-Basic-DE EBU-TT-D2EBU-TT-D-Basic-DE.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_FlashDFXP2EBU-TT-D-Basic-DE:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh FlashDFXP2EBU-TT-D-Basic-DE FlashDFXP2EBU-TT-D-Basic-DE FlashDFXP2EBU-TT-D-Basic-DE.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_STLXML_XSD:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_stlxml-xsd.sh
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+# TT-Helper modules
+tests_Helper_DFXP2TTML:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh DFXP2TTML TT-Helper/DFXP2TTML DFXP2TTML.xsl
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_Helper_GetTTMLProfile:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh GetTTMLProfile TT-Helper/GetTTMLProfile GetTTMLProfile.xsl
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_Helper_TT-Edit-List:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh TT-Edit-List TT-Helper/TT-Edit-List TT-Edit-List.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_Helper_TT-Filter-Styles-No-References:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh TT-Filter-Styles-No-References TT-Helper/TT-Filter-Styles-No-References TT-Filter-Styles-No-References.xslt
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure
+
+tests_Helper_TTML2DFXP:
+  tags:
+    - docker
+  stage: test
+  script: scf_test_driver/ci/test_standard.sh TTML2DFXP TT-Helper/TTML2DFXP TTML2DFXP.xsl
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+    # test results
+    - index.html
+    # provide test results only on failure
+    when: on_failure

misc/ci/README.md

@@ -0,0 +1,3 @@
+# CI
+This folder contains miscellaneous files related to the internal CI
+process.

misc/ci/make_reproducible_EBU-TT.xsl

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ebuttm="urn:ebu:tt:metadata" version="2.0">
+    <!-- copy unaffected nodes -->
+    <xsl:template match="@*|node()">
+        <xsl:copy>
+            <xsl:apply-templates select="@*|node()"/>
+        </xsl:copy>
+    </xsl:template>
+
+    <!-- replace document creation/revision date (current date) with fixed date -->
+    <xsl:template match="text()[parent::ebuttm:documentCreationDate or parent::ebuttm:documentRevisionDate]">2018-01-01</xsl:template>
+</xsl:stylesheet>