Skip to content

Commit

Permalink
@image支持图片列表
Browse files Browse the repository at this point in the history
  • Loading branch information
xtuhcy committed Oct 25, 2017
1 parent 8e4c3ce commit de92cf0
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 2 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.geccocrawler</groupId>
<artifactId>gecco</artifactId>
<version>1.3.0</version>
<version>1.3.1</version>

<name>Gecco</name>
<description>Easy to use lightweight web crawler</description>
Expand Down
Expand Up @@ -2,6 +2,7 @@

import java.lang.reflect.Field;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -42,12 +43,25 @@ public void render(HttpRequest request, HttpResponse response, BeanMap beanMap,
beanMap.putAll(fieldMap);
}

@SuppressWarnings("unchecked")
private Object injectImageField(HttpRequest request, BeanMap beanMap, SpiderBean bean, Field field) {
Object value = beanMap.get(field.getName());
if(value == null) {
return null;
}
String imgUrl = value.toString();
if(value instanceof Collection) {
Collection<Object> collection = (Collection<Object>)value;
for(Object item : collection) {
String imgUrl = downloadImage(request, field, item.toString());
item = imgUrl;
}
return collection;
} else {
return downloadImage(request, field, value.toString());
}
}

private String downloadImage(HttpRequest request, Field field, String imgUrl) {
if(StringUtils.isEmpty(imgUrl)) {
return imgUrl;
}
Expand Down
@@ -0,0 +1,43 @@
package com.geccocrawler.gecco.demo.images;

import java.util.List;

import com.geccocrawler.gecco.GeccoEngine;
import com.geccocrawler.gecco.annotation.Gecco;
import com.geccocrawler.gecco.annotation.HtmlField;
import com.geccocrawler.gecco.annotation.Image;
import com.geccocrawler.gecco.annotation.PipelineName;
import com.geccocrawler.gecco.pipeline.Pipeline;
import com.geccocrawler.gecco.spider.HtmlBean;

@PipelineName("imageListDemo")
@Gecco(matchUrl = "http://canlian.jiading.gov.cn/gyzc/zcxmdt/content_430614", pipelines = "imageListDemo")
public class ImageListDemo implements HtmlBean, Pipeline<ImageListDemo> {

private static final long serialVersionUID = -5583524962096502124L;

@Image
@HtmlField(cssPath = ".conTxt p img")
public List<String> pics;

public List<String> getPics() {
return pics;
}

public void setPics(List<String> pics) {
this.pics = pics;
}

@Override
public void process(ImageListDemo test) {
System.out.println(test.getPics());
}

public static void main(String[] args) {
GeccoEngine.create()
.classpath("com.geccocrawler.gecco.demo.images")
.start("http://canlian.jiading.gov.cn/gyzc/zcxmdt/content_430614")
.interval(1000)
.run();
}
}

0 comments on commit de92cf0

Please sign in to comment.