/
CrawleLogin.java
169 lines (152 loc) · 6.96 KB
/
CrawleLogin.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
package com.jamal.crawler;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
/**
* 网页采集遇到登录问题，我该怎么办？
* 豆瓣个人主页为例
* <p>
* 1、手动设置cookies
* <p>
* 2、模拟登录
* crawler
* 2019/9/29 16:32
*
* @author
**/
public class CrawleLogin {
public static void main(String[] args) throws Exception {
// 个人中心url
String user_info_url = "https://www.douban.com/people/150968577/";
// 登陆接口
String login_url = "https://accounts.douban.com/j/mobile/login/basic";
new CrawleLogin().setCookies(user_info_url);
// new CrawleLogin().jsoupLogin(login_url,user_info_url);
// new CrawleLogin().httpClientLogin(login_url,user_info_url);
}
/**
* 手动设置 cookies
* 先从网站上登录，然后查看 request headers 里面的 cookies
* @param url
* @throws IOException
*/
public void setCookies(String url) throws IOException {
Document document = Jsoup.connect(url)
// 手动设置cookies
.header("Cookie", "push_noty_num=0; push_doumail_num=0; __utmv=30149280.15096; douban-profile-remind=1; _vwo_uuid_v2=D6B70333B716FC46F4E58135BBDB4414F|684457b0372138eeedc1d3828e244a55; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1570525814%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D; _pk_ses.100001.8cb4=*; ap_v=0,6.0; __utma=30149280.933949496.1568948318.1569824493.1570525816.5; __utmc=30149280; __utmz=30149280.1570525816.5.4.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/; dbcl2=\"150968577:MMn7VZsaPqU\"; ck=oPH9; _pk_id.100001.8cb4=b366dc92e126c244.1568948315.4.1570526614.1569824689.; __utmb=30149280.7.10.1570525816")
.get();
//
if (document != null) {
// 获取豆瓣昵称节点
Element element = document.select(".info h1").first();
if (element == null) {
System.out.println("没有找到 .info h1 标签");
return;
}
// 取出豆瓣节点昵称
String userName = element.ownText();
System.out.println("豆瓣我的网名为：" + userName);
} else {
System.out.println("出错啦！！！！！");
}
}
/**
* Jsoup 模拟登录豆瓣 访问个人中心
* 在豆瓣登录时先输入一个错误的账号密码，查看到登录所需要的参数
* 先构造登录请求参数，成功后获取到cookies
* 设置request cookies，再次请求
* @param loginUrl 登录url
* @param userInfoUrl 个人中心url
* @throws IOException
*/
public void jsoupLogin(String loginUrl,String userInfoUrl) throws IOException {
// 登录数据
Map<String,String> data = new HashMap<>();
data.put("name","your_account");
data.put("password","your_password");
data.put("remember","false");
data.put("ticket","");
data.put("ck","");
Connection.Response login = Jsoup.connect(loginUrl)
.ignoreContentType(true) // 忽略类型验证
.followRedirects(false) // 禁止重定向
.postDataCharset("utf-8")
.header("Upgrade-Insecure-Requests","1")
.header("Accept","application/json")
.header("Content-Type","application/x-www-form-urlencoded")
.header("X-Requested-With","XMLHttpRequest")
.header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")
.data(data)
.method(Connection.Method.POST)
.execute();
login.charset("UTF-8");
// login 中已经获取到登录成功之后的cookies
// 构造访问个人中心的请求
Document document = Jsoup.connect(userInfoUrl)
// 取出login对象里面的cookies
.cookies(login.cookies())
.get();
if (document != null) {
Element element = document.select(".info h1").first();
if (element == null) {
System.out.println("没有找到 .info h1 标签");
return;
}
String userName = element.ownText();
System.out.println("豆瓣我的网名为：" + userName);
} else {
System.out.println("出错啦！！！！！");
}
}
/**
* httpclient 的方式模拟登录豆瓣
* httpclient 跟jsoup差不多，不同的地方在于 httpclient 有session的概念
* 在同一个httpclient 内不需要设置cookies ，会默认缓存下来
* @param loginUrl
* @param userInfoUrl
*/
public void httpClientLogin(String loginUrl,String userInfoUrl) throws Exception{
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpUriRequest login = RequestBuilder.post()
.setUri(new URI(loginUrl))// 登陆url
.setHeader("Upgrade-Insecure-Requests","1")
.setHeader("Accept","application/json")
.setHeader("Content-Type","application/x-www-form-urlencoded")
.setHeader("X-Requested-With","XMLHttpRequest")
.setHeader("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")
// 设置账号信息
.addParameter("name","your_account")
.addParameter("password","your_password")
.addParameter("remember","false")
.addParameter("ticket","")
.addParameter("ck","")
.build();
// 模拟登陆
CloseableHttpResponse response = httpclient.execute(login);
if (response.getStatusLine().getStatusCode() == 200){
// 构造访问个人中心请求
HttpGet httpGet = new HttpGet(userInfoUrl);
CloseableHttpResponse user_response = httpclient.execute(httpGet);
HttpEntity entity = user_response.getEntity();
//
String body = EntityUtils.toString(entity, "utf-8");
// 偷个懒，直接判断 缺心眼那叫单纯 是否存在字符串中
System.out.println("缺心眼那叫单纯是否查找到？"+(body.contains("缺心眼那叫单纯")));
}else {
System.out.println("httpclient 模拟登录豆瓣失败了!!!!");
}
}
}