debug: langlinks reader

wikimedia · Apr 19, 2017 · 4f0ce6c · 4f0ce6c
1 parent 1688775
commit 4f0ce6c
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 6 deletions.
diff --git a/src/main/java/org/wikipedia/citolytics/multilang/MultiLang.java b/src/main/java/org/wikipedia/citolytics/multilang/MultiLang.java
@@ -7,6 +7,7 @@
 import org.apache.flink.api.java.operators.DataSource;
 import org.apache.flink.util.Collector;
 
+import java.util.Arrays;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -58,12 +59,18 @@ public void flatMap(String s, Collector<LangLinkTuple> out) throws Exception {
 
                 if (cols.length == 3) {
                     // Match VALUES (csv split)
-                    out.collect(new LangLinkTuple(
-                            Integer.parseInt(cols[0]),
-                            cols[1].replaceAll("^'|'$", "").replace("\\'", "'"),
-                            cols[2].replaceAll("^'|'$", "").replace("\\'", "'")
-                    ));
+                    try {
+                        int pageId = Integer.parseInt(cols[0]);
 
+
+                        out.collect(new LangLinkTuple(
+                                pageId,
+                                cols[1].replaceAll("^'|'$", "").replace("\\'", "'"),
+                                cols[2].replaceAll("^'|'$", "").replace("\\'", "'")
+                        ));
+                    } catch (NumberFormatException e) {
+                        throw new Exception("Cannot parse pageID: " + cols[0] + " // cols: " + Arrays.toString(cols));
+                    }
                 } else {
                     // Match INSERT INTO statement
                     Pattern p2 = Pattern.compile("([0-9]+),'(.*?)','(.*?)'");

diff --git a/support/flink-jobs/README.md b/support/flink-jobs/README.md
@@ -154,7 +154,7 @@ The following jobs perform pre-processing or analysis tasks.
         --input $WIKI_DUMP \
         --output $SEEALSO_PATH
         
-With multi-language translation
+With multi-language translation (See [cirrusearch.md](cirrussearch.md) for lang-links extraction)
 
     $FLINK_HOME/bin/flink run -c org.wikipedia.citolytics.seealso.SeeAlsoExtrator  -p $PARALLELISM $JAR \
         --input $WIKI_DUMP \