Merge pull request #47 from nzv8fan/issue42

Fix Issue #42 to improve content identification
karussell · May 11, 2016 · cb24ab4 · cb24ab4
2 parents a5fe61e + 0326a5f
commit cb24ab4
Show file tree

Hide file tree

Showing 3 changed files with 1,142 additions and 0 deletions.
diff --git a/src/main/java/de/jetwick/snacktory/ArticleTextExtractor.java b/src/main/java/de/jetwick/snacktory/ArticleTextExtractor.java
@@ -307,6 +307,10 @@ protected int weightChildNodes(Element rootEl) {
         List<Element> pEls = new ArrayList<Element>(5);
         for (Element child : rootEl.children()) {
             String ownText = child.ownText();
+
+            // if you are on a paragraph, grab all the text including that surrounded by additional formatting.
+            if (child.tagName().equals("p")) ownText = child.text();
+
             int ownTextLength = ownText.length();
             if (ownTextLength < 20)
                 continue;

diff --git a/src/test/java/de/jetwick/snacktory/ArticleTextExtractorTest.java b/src/test/java/de/jetwick/snacktory/ArticleTextExtractorTest.java
@@ -60,6 +60,12 @@ public void testData5() throws Exception {
         assertTrue(res.getKeywords().isEmpty());
     }
 
+    @Test
+    public void testData6() throws Exception {
+        JResult res = extractor.extractContent(readFileAsString("test_data/6.html"));
+        assertEquals(res.getText(), "Acting Governor of Balkh province, Atta Mohammad Noor, said that differences between leaders of the National Unity Government (NUG) – namely President Ashraf Ghani and CEO Abdullah Abdullah— have paved the ground for mounting insecurity. Hundreds of worried relatives gathered outside Kabul hospitals on Tuesday desperate for news of loved ones following the deadly suicide bombing earlier in the day.");
+    }
+
     @Test
     public void testCNN() throws Exception {
         // http://edition.cnn.com/2011/WORLD/africa/04/06/libya.war/index.html?on.cnn=1