Merge pull request #473 from ebu/release/2.1.1

Release v2.1.1

.gitignore

@@ -109,8 +109,9 @@ gen
 # Generated bindings should not appear in the repo
 ebu_tt_live/bindings/raw
 
-# Test export files from file system carriage
+# Test and non-test export files from file system carriage
 test_export/
+export/
 
 # Node.js file
 node_modules/
@@ -125,5 +126,9 @@ docs/build/*
 !docs/build/index.html
 !docs/build/img
 
+# Test Sequences export folder
+export/
+
 # Mac OS X files
 .DS_Store
+

Makefile

@@ -1,15 +1,10 @@
-.PHONY: docs
+.PHONY: docs uiclean uibuild template initnpm uicopy
 
+all: init ui
+
 init:
 	pip install --upgrade -r requirements.txt
 	pyxbgen --binding-root=./ebu_tt_live/bindings -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd
-ifeq ("$(wildcard node_modules)","")
-	npm install nunjucks
-else 
-	npm update nunjucks
-endif
-	node_modules/nunjucks/bin/precompile ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.xml > ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.js
-	node_modules/nunjucks/bin/precompile ebu_tt_live/ui/user_input_producer/template/live_message_template.xml > ebu_tt_live/ui/user_input_producer/template/live_message_template.js
 
 initnpm:
 ifeq ("$(wildcard node_modules)","")
@@ -18,9 +13,6 @@ else
 	npm update nunjucks
 endif
 
-template:
-	node_modules/nunjucks/bin/precompile ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.xml > ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.js
-
 test:
 	python setup.py test
 
@@ -29,3 +21,24 @@ docs:
 
 bindings:
 	pyxbgen --binding-root=./ebu_tt_live/bindings -m __init__ --schema-root=./ebu_tt_live/xsd/ -r -u ebutt_all.xsd
+
+ui: uiclean uibuild
+
+uiclean:
+ifneq ("$(wildcard docs/build/ui/.)","")
+	rm -R docs/build/ui
+endif
+
+uibuild: template uicopy
+
+template: initnpm
+	node_modules/nunjucks/bin/precompile ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.xml > ebu_tt_live/ui/user_input_producer/template/user_input_producer_template.js
+	node_modules/nunjucks/bin/precompile ebu_tt_live/ui/user_input_producer/template/live_message_template.xml > ebu_tt_live/ui/user_input_producer/template/live_message_template.js
+
+uicopy:
+	mkdir -p docs/build/ui
+	cp -R ebu_tt_live/ui/user_input_producer docs/build/ui/
+	cp -R ebu_tt_live/ui/test docs/build/ui/
+	cp -R ebu_tt_live/ui/assets docs/build/ui/user_input_producer/
+	cp -R ebu_tt_live/ui/assets docs/build/ui/test/
+

docs/build/index.html

@@ -86,7 +86,7 @@ <h2>The Project</h2>
       <p>This is an open source project hosted by the <a href="https://tech.ebu.ch/home" target="_blank" rel="noopener">EBU</a> on GitHub, using Travis-CI for continuous integration and coveralls for code test coverage. The license is BSD3.</p>
       <p>This is an activity of the EBU <a href="https://tech.ebu.ch/subtitling" target="_blank" rel="noopener">Subtitles in XML</a> group.</p>
       <p>You can find details about the project, including details of how to contribute, on its <a href="https://github.com/ebu/ebu-tt-live-toolkit/" target="_blank" rel="noreferrer noopener">github page</a>.</p>
-      <p>For discussion of issues etc we have a Slack team - <a href="https://slack-invite.ebu.io/tt-lit/" target="_blank" rel="noreferrer noopener">request an invitation to sign up</a>.</p>
+      <p>For discussion of issues etc we have a Slack team - request an invitation via <a href="mailto:subtitling@ebu.ch" target="_blank" rel="noreferrer noopener">subtitling@ebu.ch</a>.</p>
       <h3>Quick links</h3>
       <p>
          <span class="q"><a href="html/index.html" target="_self" rel="noopener">documentation</a></span>

docs/source/conf.py

@@ -77,9 +77,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.1'
+version = u'2.1.1'
 # The full version, including alpha/beta/rc tags.
-release = u'2.1'
+release = u'2.1.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/configurator.rst

@@ -65,10 +65,10 @@ The more detailed options are: ::
 
     nodes
     ├─[nodeN] : a node to configure - any from node1 to node9
-    │ ├─type : ["simple-consumer" | "simple-producer" | "resequencer" | "ebuttd-encoder" | "buffer-delay" | "retiming-delay" | "distributor" | "handover"]
+    │ ├─type : ["simple-consumer" | "simple-producer" | "resequencer" | "ebuttd-encoder" | "buffer-delay" | "retiming-delay" | "distributor" | "handover" | "deduplicator"]
     │ ├─output : the output settings for the node, if applicable
     │ │ ├─carriage : the carriage mechanism to use to get incoming documents
-    │ │ │ ├─type : ["direct" | "filesystem" | "filesystem-simple" | "websocket" | "websocket-legacy"]
+    │ │ │ ├─type : ["direct" | "filesystem" | "websocket" | "websocket-legacy"]
     │ │ │ └─[output carriage type-dependent options - see below]
     │ │ └─adapters : see below
     │ ├─input : the input settings for the node, if applicable
@@ -79,7 +79,7 @@ The more detailed options are: ::
     │ ├─id : identifier for the node, defaults to a value based on the node's type.
     │ └─[node type-dependent options - see below]
     backend
-    └─type: ["twisted" (default) | "dummy"]    
+    └─type: ["twisted" (default) | "dummy"]
 
 Node type dependent options for [nodeN] : ::
 
@@ -120,6 +120,9 @@ Node type dependent options for [nodeN] : ::
    ├─authors_group_identifier : the authors' group to follow, default "AuthorsGroup1"
    └─sequence_identifier : sequence identifier, default "HandoverSequence1"
 
+   type="deduplicator"
+   └─sequence_identifier : sequence identifier, default "DeDuplicated1"
+
    type="distributor" : No options
 
 Output carriage type dependent options for "carriage": ::
@@ -128,12 +131,11 @@ Output carriage type dependent options for "carriage": ::
    └─id : id of the 'pipe' to write to, default "default"
 
    type="filesystem"
-   └─folder : The output folder/directory. Folder is created if it does not exist. Existing files are overwritten, default "./export"
-
-   type="filesystem-simple"
    ├─folder : The output folder/directory. Folder is created if it does not exist. Existing files are overwritten, default "./export"
-   ├─rotating_buf : Rotating buffer size. This will keep the last N number of files created in the folder., default 0
-   └─filename_pattern : File name pattern. It needs to contain {counter} format parameter, default "export-{counter}.xml"
+   ├─rotating_buf : Rotating buffer size. This will keep the last N number of files created in the folder or all if 0, default 0
+   ├─suppress_manifest : Whether to suppress writing of a manifest file (e.g. for EBU-TT-D output). Default False
+   ├─message_filename_pattern : File name pattern for message documents or EBU-TT-D documents. It can contain {sequence_identifier} and {counter} format parameters, default "{sequence_identifier}_msg_{counter}.xml" 
+   └─filename_pattern : File name pattern for EBU-TT-Live documents. It needs to contain {counter} format parameter, which will be populated with the sequence number. Default "{sequence_identifier}_{counter}.xml"
 
    type="websocket"
    ├─proxy : HTTP proxy in format ADDR:PORT
@@ -153,11 +155,6 @@ Input carriage type dependent options for "carriage": ::
    ├─manifest_file : The timing manifest file for importing files. Files are required to be in the same folder as the manifest file.
    └─tail : Keep the manifest open and wait for new input much like UNIX's tail -f command
 
-   type="filesystem-simple"
-   ├─folder : The output folder/directory. Folder is created if it does not exist. Existing files are overwritten, default "./export"
-   ├─rotating_buf : Rotating buffer size. This will keep the last N number of files created in the folder., default 0
-   └─filename_pattern : File name pattern. It needs to contain {counter} format parameter, default "export-{counter}.xml"
-
    type="websocket"
    ├─proxy : HTTP proxy in format ADDR:PORT
    ├─listen : Socket to listen on for /publish connection requests i.e: ws://ADDR:PORT, default "ws://localhost:9001"

docs/source/deduplication.rst

@@ -0,0 +1,44 @@
+DeDuplication of EBU-TT-Live documents
+======================================
+
+When documents are ReSequenced, duplication of ``style`` and ``region`` elements
+and attributes can occur. To address this, the DeDuplicator node processes the
+document(s) with the
+:py:func:`ebu_tt_live.node.deduplicator.DeDuplicatorNode.remove_duplication` function.
+
+After copying ``styling`` and ``layout`` into a ``list()`` and setting them up for new
+``style`` and ``region`` elements, respectively, the ``list()`` is passed through the
+:py:func:`ebu_tt_live.node.deduplicator.DeDuplicatorNode.CollateUniqueVals` function.
+
+Because ``style`` and ``region`` elements can have ``style`` attributes, these
+are deduplicated first. At this stage, it's possible that where two identical elements
+that differed only in their style references, these may end up looking the same.
+ Each element is then passed through the
+:py:class:`ebu_tt_live.node.deduplicator.ComparableElement` class, which processes
+each attribute, omitting the ``xml:id`` and using the
+:py:func:`ebu_tt_live.node.deduplicator.ReplaceNone` function to replace empty
+attributes with a non-legal character to avoid collisions, and assigns a hash
+to each element. The hash is then stored in ``old_id_dict`` as a key-value pair,
+where the ``xml:id`` is the ``key`` and the hash is the ``value``. The hash is also stored
+in ``hash_dict`` where the hash is the ``key``, and the ``value`` is the contents of the element.
+
+The ``list()`` is then passed through the :py:func:`ebu_tt_live.node.deduplicator.DeDuplicatorNode.AppendNewElements`
+function, which takes in the list of elements, the path to the parent element
+(``styling`` or ``layout``), ``old_id_dict``, ``new_id_dict`` and ``hash_dict``
+
+The function iterates through the key-value pairs of ``hash_dict`` and the contents
+of the list of elements; where the ``xml:id`` of both matches, the element is appended to
+the parent element. The hash and ``xml:id`` is then stored in ``new_id_dict``,
+where the hash is the ``key`` and the ``xml:id`` is the ``value``.
+
+In the final step, before emitting the document, the document as it is at this stage is passed
+through the :py:class:`ebu_tt_live.node.deduplicator.ReplaceStylesAndRegions` class.
+This utilises RecursiveOperation i.e. this class is used to recursively fix
+the style and region attribute values, by addressing where a ``style`` or ``region``
+attribute has been declared, and replaces the reference to the old ``xml:id`` with
+the new one stored in ``new_id_dict``, which is done by taking the reference to
+the old ``xml:id``, matching it to the key in ``old_id_dict`` to find the hash
+``value``, then matching the hash to the ``key`` in ``new_id_dict`` to get the
+new ``xml:id`` reference. While doing this, it also deduplicates instances where
+multiple ``style`` attributes have been referenced, removing duplicates while
+maintaining the hierarchy in which they were declared.

docs/source/ebu_tt_live.node.rst

@@ -72,3 +72,11 @@ node Package
     :members:
     :undoc-members:
     :show-inheritance:
+
+:mod:`deduplicator` Module
+----------------------
+
+.. automodule:: ebu_tt_live.node.deduplicator
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/overview.rst

@@ -3,9 +3,9 @@ Overview of the EBU-TT live toolkit.
 
 This page is a short introduction to using those scripts that demonstrate toolkit components. For more details and information about other scripts, follow the links below.
 
-To run the scripts, you will need to first set up the environment and build the code. Please follow the instructions in https://github.com/ebu/ebu-tt-live-toolkit/blob/master/README.md. 
+To run the scripts, you will need to first set up the environment and build the code. Please follow the instructions in https://github.com/ebu/ebu-tt-live-toolkit/blob/master/README.md.
 
-Not all components are implemented yet - see https://github.com/ebu/ebu-tt-live-toolkit/wiki/Components for a list of all components. This page will be updated as more scripts are added. 
+Not all components are implemented yet - see https://github.com/ebu/ebu-tt-live-toolkit/wiki/Components for a list of all components. This page will be updated as more scripts are added.
 
 The components mimic the nodes and carriage mechanisms defined in the specification. Producer components create documents; consumer components consume them. Each script combines a carriage mechanism implementation with a processing node to create code that can operate as a node.
 
@@ -17,4 +17,5 @@ The components mimic the nodes and carriage mechanisms defined in the specificat
     user_input_producer
     timing_resolution
     segmentation
+    deduplication
     conversion

docs/source/scripts_and_their_functions.rst

@@ -22,7 +22,8 @@ into a sequence of EBU-TT Live documents using natural language processing. Use
 
 The default carriage mechanism is WebSocket, so you will need to listen to
 ``ws://127.0.0.1:9000``. Conveniently, we've created an HTML page that does just
-that. After you launch the Simple Producer, open ``test.html`` in your
+that. After you launch the Simple Producer, open `docs/build/ui/test/index.html <../ui/test/index.html>`_
+or the `current release pre-built page <http://ebu.github.io/ebu-tt-live-toolkit/ui/test/>`_ in your
 browser. The 'Broadcast message' field should be populated with the correct
 address (``ws://localhost:9000``). Click 'Connect' and then 'Subscribe'. You can
 also change the identifier for the sequence. The documents should appear in the
@@ -56,7 +57,8 @@ Distributor or a Handover Manager Node. First, with your virtual environment
 activated and the code built, start one, with a command line such as  ``ebu-run
 --admin.conf ebu_tt_live/examples/config/user_input_producer_consumer.json`` -
 this one runs a simple consumer. Then, in your browser, open
-``ebu_tt_live/ui/user_input_producer/user_input_producer.html`` and click
+`docs/build/ui/user_input_producer/index.html <../ui/user_input_producer/index.html>`_ or the
+`current release pre-built page <http://ebu.github.io/ebu-tt-live-toolkit/ui/user_input_producer/>`_ and click
 'Connect'. Select the sending mode (manual, scheduled or asynchronous). You
 should see the documents arriving in the command line window where the simple
 consumer is listening. See detailed instructions here:
@@ -98,6 +100,13 @@ required. The Buffer Delay Node is primarily intended for delaying implicitly
 timed documents for resynchronisation. Use ``ebu-run`` to start this script, for
 example ``ebu-run --admin.conf=ebu_tt_live/examples/config/buffer_delay.json``
 
+DeDuplicator Node
+-----------------
+This node addresses instances where ``style`` and ``region`` elements and
+attributes are duplicated.
+For the default configuration of the node, see:
+``ebu-run --admin.conf=ebu_tt_live/examples/config/deduplicator_fs.json``
+
 Retiming Delay Node
 -------------------
 This script modifies the times within each Document and issues them without
@@ -106,7 +115,7 @@ The times are modified such that all of the computed begin and end times within
 the document are increased by a non-negative fixed delay offset period. The
 Retiming Delay Node is primarily intended for delaying explicitly timed
 documents. Use ``ebu-run`` to start this script, for example ``ebu-run
---admin.conf=ebu_tt_live/examples/config/retiming_delay.json.`` 
+--admin.conf=ebu_tt_live/examples/config/retiming_delay.json.``
 
 EBU-TT-D Encoder
 ----------------

docs/source/user_input_producer.rst

@@ -15,8 +15,8 @@ listen for /publish requests - for example ``ebu-run --admin.conf
 ebu_tt_live/examples/config/user_input_producer_consumer.json`` will run the
 simplest user-input-consumer script listening for a connection on the local host
 port 9001. Then open the
-``ebu_tt_live/ui/user_input_producer/user_input_producer.html`` file in your
-browser.
+`docs/build/ui/user_input_producer/index.html <../ui/user_input_producer/index.html>`_ file in your
+browser, or open the `current release pre-built page <http://ebu.github.io/ebu-tt-live-toolkit/ui/user_input_producer/>`_.
 
 If you have no configured sequences or you need to create a new one, click
 "Create a new sequence", enter the sequence number, what time base and clock
@@ -61,7 +61,7 @@ transport mechanism, follow these steps:
 
 * Start the Handover Manager: ``ebu-run --admin.conf=ebu_tt_live/examples/config/user_input_producer_handover.json``
 
-* Create two (or more) instances of the User Input Producer by opening ``ebu_tt_live/ui/user_input_producer/user_input_producer.html`` in multiple browser tabs/windows.
+* Create two (or more) instances of the User Input Producer by opening `docs/build/ui/user_input_producer/index.html <../ui/user_input_producer/index.html>`_ or the `current release pre-built page <http://ebu.github.io/ebu-tt-live-toolkit/ui/user_input_producer/>`_ in multiple browser tabs/windows.
 
 * In each instance of the UIP, create a sequence with a different sequence identifier but with identical authors group identifier. If you use the default configuration, this is "TestGroup1".
 
@@ -84,7 +84,7 @@ in the Control Request field.
 
 .. note:: More demo scenario examples :
 
-    * Run ``ebu-run --admin.conf ebu_tt_live/examples/config/user_input_producer_dist_consumers.json``. This configuration runs a distributor node and two consumers, one for TestSequence1 and the other for TestSequence2. Open the ``ui/user_input_producer/user_input_producer.html`` page and create or select TestSequence1, and connect it to the distributor node via websocket (click "connect" button). When you send a document the consumer will log the new document. Open another user input web page and connect it with TestSequence2. The second consumer receives this document. Now documents can be sent by both web pages simultaneously and will be routed to the correct consumer by the Distributor node.
+    * Run ``ebu-run --admin.conf ebu_tt_live/examples/config/user_input_producer_dist_consumers.json``. This configuration runs a distributor node and two consumers, one for TestSequence1 and the other for TestSequence2. Open the `docs/build/ui/user_input_producer/index.html <../ui/user_input_producer/index.html>`_ page or the `current release pre-built page <http://ebu.github.io/ebu-tt-live-toolkit/ui/user_input_producer/>`_ and create or select TestSequence1, and connect it to the distributor node via websocket (click "connect" button). When you send a document the consumer will log the new document. Open another user input web page and connect it with TestSequence2. The second consumer receives this document. Now documents can be sent by both web pages simultaneously and will be routed to the correct consumer by the Distributor node.
 
 .. note:: Demo scenario examples that need some better documentation :
 

docs/source/validation_framework.rst

@@ -6,7 +6,7 @@ a pretty good job validating the syntax but it does not provide any tooling to p
 be described in XSD 1.0 (PyXB does not understand XSD 1.1 and xpath assertions).
 
 Example:
-    if tt:tt has timeBase=clock smpteTimingType must not be present anywhere in the document.
+    If ttp:timebase="clock" then all begin and end attributes of timed elements shall be of type ebuttdt:clockTimingType.
 
 Elements/types are independent and according to the principles of OOP they do not know much about one another.
 In order to provide semantic validation there is need for a semantic validation context.

ebu-debug.py

@@ -0,0 +1,3 @@
+from ebu_tt_live.scripts import ebu_run
+
+ebu_run.main()

ebu_tt_live/bindings/__init__.py

@@ -440,6 +440,7 @@ class LiveStyledElementMixin(StyledElementMixin):
 
 # EBU TT Live element types
 # =========================
+# NOTE: Some of the code below includes handling of SMPTE time base, which was removed from version 1.0 of the specification.
 
 
 class tt_type(SemanticDocumentMixin, raw.tt_type):
@@ -652,7 +653,6 @@ def get_timing_type(self, timedelta_in):
         if self.timeBase == 'smpte':
             return ebuttdt.SMPTETimingType(timedelta_in)
 
-
 raw.tt_type._SetSupersedingClass(tt_type)
 
 
@@ -823,7 +823,6 @@ def content_to_string(self, begin=None, end=None):
 
 raw.br_type._SetSupersedingClass(br_type)
 
-
 class div_type(ContentContainerMixin, IDMixin, RegionedElementMixin, LiveStyledElementMixin, TimingValidationMixin,
                SemanticValidationMixin, raw.div_type):