Merge branch 'master' into 2.0.x

* master: #1322 - Upgrade to OpenNLP 1.9.1 #1308 - integrate mystem #1327 - Update LIF support #1327 - Update LIF support #1329 - Span annotations with slot features may disappear from WebAnno TSV #1329 - Span annotations with slot features may disappear from WebAnno TSV #1329 - Span annotations with slot features may disappear from WebAnno TSV #1327 - Update LIF support #1323 - File extension generated by BinaryCasWriter does not contain dot #858 - Out-of-tagset tags should map to the generic type #1239 - Rename NYTCollectionReader to NitfReader #858 - Out-of-tagset tags should map to the generic type #1317 - Standard parameter to disable type mapping No issue. If a DKProTextContext is available, then TestRunner generates an XMI file from the processed data and stores it in the test output folder. No issue - Log names of files with license issues to the console. #1160 - Better support for CoNLL-U v2 (1.11.0) % Conflicts: % dkpro-core-asl/pom.xml
dkpro · Mar 19, 2019 · def675d · def675d
2 parents 509414c + 9e2c7ef
commit def675d
Show file tree

Hide file tree

Showing 134 changed files with 6,451 additions and 811 deletions.
diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/bn-utpal-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/bn-utpal-pos.map
@@ -43,5 +43,5 @@ SYM=POS_SYM
 RDP=POS_X
 # UNK  Unknown
 UNK=POS_X
-*=POS_X
+*=POS
 
diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-stts-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-stts-pos.map
@@ -12,7 +12,7 @@ __META_SOURCE_URL__=http://www.ims.uni-stuttgart.de/forschung/ressourcen/lexika/
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Catch-all rule
-*=POS_X
+*=POS
 #
 # $*LRB* 
 # -

diff --git a/...ain/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-tiger-rftagger-pos.map b/...ain/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/de-tiger-rftagger-pos.map
@@ -38,4 +38,4 @@ VINF=POS_VERB
 SYM=POS_PUNCT
 SENT=POS_PUNCT
 
-*=POS_X
+*=POS
diff --git a/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-arktweet-pos.map b/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-arktweet-pos.map
@@ -3,7 +3,7 @@ __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 $=POS_NUM
 &=POS_CONJ
-*=POS_X
+*=POS
 ,=POS_PUNCT
 @=tweet.POS_AT
 A=POS_ADJ

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-brown-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-brown-pos.map
@@ -10,7 +10,7 @@ __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 # right paren
 )=POS_X
 # not, n't
-#*=POS_X
+#*=POS
 # dash
 -=POS_X
 # comma
@@ -185,4 +185,4 @@ NEG=POS_ADV
 NNSG=POS_NOUN
 NR$=POS_NOUN
 
-*=POS_X
+*=POS
diff --git a/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-browntei-pos.map b/.../src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-browntei-pos.map
@@ -86,4 +86,4 @@ NRg=POS_NOUN
 RBT=POS_ADV
 UH=POS_PART
 WQL=POS_ADV
-*=POS_X
+*=POS
diff --git a/...ph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-c5-pos.map b/...ph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-c5-pos.map
@@ -6,7 +6,7 @@ __META_SOURCE_URL__=http://www.natcorp.ox.ac.uk/docs/gramtag.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Default mapping
-*=POS_X
+*=POS
 
 # AJ0	Adjective (general or positive) (e.g. good, old, beautiful)
 AJ0=POS_ADJ

diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-lbj-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-lbj-pos.map
@@ -5,7 +5,7 @@
 
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # punctuation mark, comma
 ,=POS_PUNCT
 -=POS_PUNCT

diff --git a/...l/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-medpost-pos.map b/...l/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-medpost-pos.map
@@ -124,4 +124,4 @@ VVGN=POS_VERB
 # ” right quote 
 ''=POS_PUNCT
 
-*=POS_X
+*=POS
diff --git a/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-emory-pos.map b/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-emory-pos.map
@@ -7,7 +7,7 @@
 
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # punctuation mark, comma
 ,=POS_PUNCT
 -=POS_PUNCT

diff --git a/...sl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-tt-pos.map b/...sl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/en-ptb-tt-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=http://faculty.washington.edu/dillon/GramResources/penntable.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 ,=POS_PUNCT
 -=POS_PUNCT
 .=POS_PUNCT

diff --git a/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/es-conll2009-pos.map b/...src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/es-conll2009-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=http://clic.ub.edu/corpus/webfm_send/18
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 #Adjective
 a=POS_ADJ
 #Conjuction

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/et-tartu-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/et-tartu-pos.map
@@ -584,4 +584,4 @@ Z.Osq=POS_PUNCT
 Z.Csq=POS_PUNCT
 Z.Sla=POS_PUNCT
 T=POS_X
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-pos.map
@@ -67,5 +67,5 @@ V_COP=POS_VERB
 V_PRS=POS_VERB
 # V_SUB - Subjunctive verb
 V_SUB=POS_VERB
-*=POS_X
+*=POS
 
diff --git a/...c/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-reduced-pos.map b/...c/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fa-upc-reduced-pos.map
@@ -39,4 +39,4 @@ STPOS=POS_X
 STR=POS_X
 # V - Verb
 V=POS_VERB
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-ftb-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-ftb-pos.map
@@ -35,4 +35,4 @@ PUNC=POS_PUNCT
 \:=POS_PUNCT
 # V
 V=POS_VERB
-*=POS_X
+*=POS
diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-stein-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/fr-stein-pos.map
@@ -7,7 +7,7 @@
 __META_SOURCE_URL__=http://www.ims.uni-stuttgart.de/~schmid/french-tagset.html
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # ABR	abreviation
 ABR=POS_X
 # ADJ	adjective

diff --git a/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-stein-pos.map b/...asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-stein-pos.map
@@ -7,7 +7,7 @@
 __META_SOURCE_URL__=ftp://ftp.ims.uni-stuttgart.de/pub/corpora/italian-tagset.txt
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 # ABR	abbreviation
 ABR=POS_X
 # ADJ	adjective

diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-tanl-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/it-tanl-pos.map
@@ -5,7 +5,7 @@
 __META_SOURCE_URL__=Source: http://medialab.di.unipi.it/wiki/Tanl_POS_Tagset
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
-*=POS_X
+*=POS
 APn=POS_ADJ
 APp=POS_ADJ
 APs=POS_ADJ

diff --git a/...morph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/ru-pos.map b/...morph-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/ru-pos.map
@@ -1,5 +1,5 @@
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
-*=POS_X
+*=POS
 A-NUM=POS_ADJ
 A-PRO=POS_PRON
 A=POS_ADJ

diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/sv-suc-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/sv-suc-pos.map
@@ -62,4 +62,4 @@ MID=POS_PUNCT
 PAD=POS_PUNCT
 
 # Catch-all rule
-*=POS_X
+*=POS
diff --git a/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-ctb-pos.map b/...h-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-ctb-pos.map
@@ -75,4 +75,4 @@ VV=POS_VERB
 # X
 X=POS_X
 
-*=POS_X
+*=POS
diff --git a/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-lcmc-pos.map b/...-asl/src/main/resources/de/tudarmstadt/ukp/dkpro/core/api/lexmorph/tagset/zh-lcmc-pos.map
@@ -6,7 +6,7 @@ __META_SOURCE_URL__=http://www.lancs.ac.uk/fass/projects/corpus/LCMC/lcmc/lcmc_t
 __META_TYPE_BASE__=de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.
 
 # Catch-all rule
-*=POS_X
+*=POS
 
 # a          adjective
 a=POS_ADJ

diff --git a/...er-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/parameter/ComponentParameters.java b/...er-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/parameter/ComponentParameters.java
@@ -312,6 +312,13 @@ else if (value < 0) {
     // Not using the type constants here because they are capitalized for use with verbs
     // =============================================================================================
 
+    public static final String DEFAULT_MAPPING_ENABLED = "true";
+
+    /**
+     * Enable/disable type mapping.
+     */
+    public static final String PARAM_MAPPING_ENABLED = "mappingEnabled";
+
     /**
      * Location of the mapping file for part-of-speech tags to UIMA types.
      */

diff --git a/...pi-parameter-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/parameter/MimeTypes.java b/...pi-parameter-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/parameter/MimeTypes.java
@@ -28,7 +28,7 @@ public final class MimeTypes
     public final static String APPLICATION_VND_XMI_XML = "application/vnd.xmi+xml";
 
 
-    // DKPro application types
+    // DKPro Core application types
     public final static String APPLICATION_X_ANCORA_XML = "application/x.org.dkpro.ancora+xml";
     public final static String APPLICATION_X_BNC = "application/x.org.dkpro.bnc+xml";
     public final static String APPLICATION_X_BRAT = "application/x.org.dkpro.brat";
@@ -67,7 +67,7 @@ public final class MimeTypes
     // Non-standard text types
     public final static String TEXT_TCF = "text/tcf+xml";
 
-    // DKPro text types
+    // DKPro Core text types
     public final static String TEXT_X_CONLL_2000 = "text/x.org.dkpro.conll-2000";
     public final static String TEXT_X_CONLL_2002 = "text/x.org.dkpro.conll-2002";
     public final static String TEXT_X_CONLL_2003 = "text/x.org.dkpro.conll-2003";
@@ -84,6 +84,7 @@ public final class MimeTypes
     public final static String TEXT_X_PTB_CHUNKED = "text/x.org.dkpro.ptb-chunked";
     public final static String TEXT_X_PTB_COMBINED = "text/x.org.dkpro.ptb-combined";
     public final static String TEXT_X_REUTERS21578 = "text/x.org.dkpro.reuters21578";
+    public final static String TEXT_X_WEBANNO_TSV3 = "text/x.org.dkpro.webanno-tsv3";
 
     // OpenNLP model types
     public final static String APPLICATION_X_OPENNLP_CHUNK = "application/x.org.dkpro.core.opennlp.chunk";

diff --git a/...ources-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/resources/MappingProvider.java b/...ources-asl/src/main/java/de/tudarmstadt/ukp/dkpro/core/api/resources/MappingProvider.java
@@ -39,6 +39,12 @@
 
 public class MappingProvider extends CasConfigurableProviderBase<Map<String, String>>
 {
+    /**
+     * Flag indicating whether the entire mapping mechanism should be skipped and the default base
+     * type should always be applied.
+     */
+    public static final String MAPPING_ENABLED = "mappingEnabled";
+
     public static final String BASE_TYPE = "baseType";
 
     private TypeSystem typeSystem;
@@ -64,6 +70,13 @@ public void configure(CAS aCas) throws AnalysisEngineProcessException
     {
         typeSystem = aCas.getTypeSystem();
 
+        // If mapping is disabled, then we simply skip the loading of the mapping and pretend we
+        // didn't find a mapping.
+        if ("false".equalsIgnoreCase(getOverride(MAPPING_ENABLED))) {
+            notFound = true;
+            return;
+        }
+
         // Tag mappings can exist independently from the type mappings because tag mappings
         // are configured in the model metadata
         tagMappings = new HashMap<>();