Skip to content

Commit

Permalink
Merge pull request #1107 from hooyantsing/develop
Browse files Browse the repository at this point in the history 
修复 HtmlCleaner 无法正常解析 tr 和 td 标签的问题
  • Loading branch information
@sutra
sutra committed Feb 4, 2023
2 parents a266df4 + 08f4a40 commit f47038d
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 41 deletions.
10 changes: 2 additions & 8 deletions webmagic-core/src/main/java/us/codecraft/webmagic/selector/BaseElementSelector.java
View file
Expand Up @@ -3,6 +3,7 @@
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import us.codecraft.webmagic.utils.BaseSelectorUtils;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -13,16 +14,9 @@
*/
public abstract class BaseElementSelector implements Selector, ElementSelector {
private Document parse(String text) {
if (text == null) {
return null;
}

// Jsoup could not parse <tr></tr> or <td></td> tag directly
// https://stackoverflow.com/questions/63607740/jsoup-couldnt-parse-tr-tag
if ((text.startsWith("<tr>") && text.endsWith("</tr>"))
|| (text.startsWith("<td>") && text.endsWith("</td>"))) {
text = "<table>" + text + "</table>";
}
text = BaseSelectorUtils.preParse(text);
return Jsoup.parse(text);
}

Expand Down
23 changes: 23 additions & 0 deletions webmagic-core/src/main/java/us/codecraft/webmagic/utils/BaseSelectorUtils.java
View file
@@ -0,0 +1,23 @@
package us.codecraft.webmagic.utils;

/**
* @author hooy
*/
public class BaseSelectorUtils {

/**
* Jsoup/HtmlCleaner could not parse "tr" or "td" tag directly
* https://stackoverflow.com/questions/63607740/jsoup-couldnt-parse-tr-tag
*
* @param text - the html string
* @return text
*/
public static String preParse(String text) {
if (((text.startsWith("<tr>") || text.startsWith("<tr ")) && text.endsWith("</tr>"))
|| ((text.startsWith("<td>") || text.startsWith("<td ")) && text.endsWith("</td>"))) {
text = "<table>" + text + "</table>";
}
return text;
}

}
28 changes: 16 additions & 12 deletions webmagic-saxon/src/main/java/us/codecraft/webmagic/selector/Xpath2Selector.java
View file
Expand Up @@ -8,6 +8,7 @@
import java.util.concurrent.ConcurrentHashMap;

import javax.xml.namespace.NamespaceContext;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
Expand All @@ -29,13 +30,14 @@

import net.sf.saxon.lib.NamespaceConstant;
import net.sf.saxon.xpath.XPathEvaluator;
import us.codecraft.webmagic.utils.BaseSelectorUtils;

/**
* 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。<br>
*
* @author code4crafter@gmail.com <br>
* Date: 13-4-21
* Time: 上午9:39
* Date: 13-4-21
* Time: 上午9:39
*/
public class Xpath2Selector implements Selector {

Expand Down Expand Up @@ -111,14 +113,11 @@ private void init() throws XPathExpressionException {
@Override
public String select(String text) {
try {
HtmlCleaner htmlCleaner = new HtmlCleaner();
TagNode tagNode = htmlCleaner.clean(text);
Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
Object result;
try {
result = xPathExpression.evaluate(document, XPathConstants.NODESET);
result = xPathExpression.evaluate(parse(text), XPathConstants.NODESET);
} catch (XPathExpressionException e) {
result = xPathExpression.evaluate(document, XPathConstants.STRING);
result = xPathExpression.evaluate(parse(text), XPathConstants.STRING);
}
if (result instanceof NodeList) {
NodeList nodeList = (NodeList) result;
Expand Down Expand Up @@ -147,14 +146,11 @@ public String select(String text) {
public List<String> selectList(String text) {
List<String> results = new ArrayList<String>();
try {
HtmlCleaner htmlCleaner = new HtmlCleaner();
TagNode tagNode = htmlCleaner.clean(text);
Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
Object result;
try {
result = xPathExpression.evaluate(document, XPathConstants.NODESET);
result = xPathExpression.evaluate(parse(text), XPathConstants.NODESET);
} catch (XPathExpressionException e) {
result = xPathExpression.evaluate(document, XPathConstants.STRING);
result = xPathExpression.evaluate(parse(text), XPathConstants.STRING);
}
if (result instanceof NodeList) {
NodeList nodeList = (NodeList) result;
Expand All @@ -179,4 +175,12 @@ public List<String> selectList(String text) {
}
return results;
}

private Document parse(String text) throws ParserConfigurationException {
// HtmlCleaner could not parse <tr></tr> or <td></td> tag directly
text = BaseSelectorUtils.preParse(text);
HtmlCleaner htmlCleaner = new HtmlCleaner();
TagNode tagNode = htmlCleaner.clean(text);
return new DomSerializer(new CleanerProperties()).createDOM(tagNode);
}
}
62 changes: 41 additions & 21 deletions webmagic-saxon/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java
View file
Expand Up @@ -11,6 +11,9 @@
import org.junit.Ignore;
import org.junit.Test;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.xsoup.XPathEvaluator;
import us.codecraft.xsoup.Xsoup;

Expand Down Expand Up @@ -1385,35 +1388,52 @@ public void testXpath2Selector() {
Assert.assertEquals("http://www.oschina.net/", selectList.get(0));
}

@Ignore("test parse <table> <tr> <td> tag")
@Test
public void htmlCleanerParseTest() {
Spider.create(new RuoxiaPageProcessor()).addUrl("http://www.ruoxia.com/top/dianji/month").thread(1).run();
}

class RuoxiaPageProcessor implements PageProcessor {
@Override
public void process(Page page) {
List<String> items = new Xpath2Selector("//div[@class=\"bd\"]//tbody/tr").selectList(page.getRawText());
for (String item : items) {
String name = new Xpath2Selector("//td[3]/div/a[1]/text()").select(item);
System.out.println(name);
}
}
}

@Ignore("take long time")
@Test
public void performanceTest() {
Xpath2Selector xpath2Selector = new Xpath2Selector("//a");
long time =System.currentTimeMillis();
long time = System.currentTimeMillis();
for (int i = 0; i < 1000; i++) {
xpath2Selector.selectList(html);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

XpathSelector xpathSelector = new XpathSelector("//a");
time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 1000; i++) {
xpathSelector.selectList(html);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 1000; i++) {
xpath2Selector.selectList(html);
}
System.out.println(System.currentTimeMillis() - time);

CssSelector cssSelector = new CssSelector("a");
time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 1000; i++) {
cssSelector.selectList(html);
}
System.out.println("css "+(System.currentTimeMillis()-time));
System.out.println("css " + (System.currentTimeMillis() - time));
}

@Ignore("take long time")
Expand All @@ -1425,54 +1445,54 @@ public void parserPerformanceTest() throws XPatherException {
TagNode tagNode = htmlCleaner.clean(html);
Document document = Jsoup.parse(html);

long time =System.currentTimeMillis();
long time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
htmlCleaner.clean(html);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
tagNode.evaluateXPath("//a");
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

System.out.println("=============");

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
Jsoup.parse(html);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
document.select("a");
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

System.out.println("=============");

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
htmlCleaner.clean(html);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
tagNode.evaluateXPath("//a");
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

System.out.println("=============");

XPathEvaluator compile = Xsoup.compile("//a");
time =System.currentTimeMillis();
time = System.currentTimeMillis();
for (int i = 0; i < 2000; i++) {
compile.evaluate(document);
}
System.out.println(System.currentTimeMillis()-time);
System.out.println(System.currentTimeMillis() - time);

}

Expand Down

0 comments on commit f47038d

Please sign in to comment.