From de92cf074fd9877b14aa031cf70d25f260a8259f Mon Sep 17 00:00:00 2001 From: xtuhcy Date: Wed, 25 Oct 2017 10:12:25 +0800 Subject: [PATCH] =?UTF-8?q?@Image=E6=94=AF=E6=8C=81=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- .../spider/render/html/ImageFieldRender.java | 16 ++++++- .../gecco/demo/images/ImageListDemo.java | 43 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/geccocrawler/gecco/demo/images/ImageListDemo.java diff --git a/pom.xml b/pom.xml index 832949e5..48b8d4ac 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.geccocrawler gecco - 1.3.0 + 1.3.1 Gecco Easy to use lightweight web crawler diff --git a/src/main/java/com/geccocrawler/gecco/spider/render/html/ImageFieldRender.java b/src/main/java/com/geccocrawler/gecco/spider/render/html/ImageFieldRender.java index ad3dbc95..2aad0ebe 100644 --- a/src/main/java/com/geccocrawler/gecco/spider/render/html/ImageFieldRender.java +++ b/src/main/java/com/geccocrawler/gecco/spider/render/html/ImageFieldRender.java @@ -2,6 +2,7 @@ import java.lang.reflect.Field; import java.net.URLEncoder; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -42,12 +43,25 @@ public void render(HttpRequest request, HttpResponse response, BeanMap beanMap, beanMap.putAll(fieldMap); } + @SuppressWarnings("unchecked") private Object injectImageField(HttpRequest request, BeanMap beanMap, SpiderBean bean, Field field) { Object value = beanMap.get(field.getName()); if(value == null) { return null; } - String imgUrl = value.toString(); + if(value instanceof Collection) { + Collection collection = (Collection)value; + for(Object item : collection) { + String imgUrl = downloadImage(request, field, item.toString()); + item = imgUrl; + } + return collection; + } else { + return downloadImage(request, field, value.toString()); + } + } + + private String downloadImage(HttpRequest request, Field field, String imgUrl) { if(StringUtils.isEmpty(imgUrl)) { return imgUrl; } diff --git a/src/test/java/com/geccocrawler/gecco/demo/images/ImageListDemo.java b/src/test/java/com/geccocrawler/gecco/demo/images/ImageListDemo.java new file mode 100644 index 00000000..96185399 --- /dev/null +++ b/src/test/java/com/geccocrawler/gecco/demo/images/ImageListDemo.java @@ -0,0 +1,43 @@ +package com.geccocrawler.gecco.demo.images; + +import java.util.List; + +import com.geccocrawler.gecco.GeccoEngine; +import com.geccocrawler.gecco.annotation.Gecco; +import com.geccocrawler.gecco.annotation.HtmlField; +import com.geccocrawler.gecco.annotation.Image; +import com.geccocrawler.gecco.annotation.PipelineName; +import com.geccocrawler.gecco.pipeline.Pipeline; +import com.geccocrawler.gecco.spider.HtmlBean; + +@PipelineName("imageListDemo") +@Gecco(matchUrl = "http://canlian.jiading.gov.cn/gyzc/zcxmdt/content_430614", pipelines = "imageListDemo") +public class ImageListDemo implements HtmlBean, Pipeline { + + private static final long serialVersionUID = -5583524962096502124L; + + @Image + @HtmlField(cssPath = ".conTxt p img") + public List pics; + + public List getPics() { + return pics; + } + + public void setPics(List pics) { + this.pics = pics; + } + + @Override + public void process(ImageListDemo test) { + System.out.println(test.getPics()); + } + + public static void main(String[] args) { + GeccoEngine.create() + .classpath("com.geccocrawler.gecco.demo.images") + .start("http://canlian.jiading.gov.cn/gyzc/zcxmdt/content_430614") + .interval(1000) + .run(); + } +}