forked from tanjiti/perl_tools
/
isProxyOK.pl
executable file
·227 lines (177 loc) · 6.79 KB
/
isProxyOK.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
#!/usr/local/bin/perl
use strict;
use warnings;
use LWP::UserAgent;
use feature qw(switch say);
use Getopt::Long;
use Carp;
use Term::ANSIColor qw(:constants);
use Time::HiRes qw(time);
use URI::Split qw(uri_split);
local $Term::ANSIColor::AUTORESET = 1;
my $help = q{};
my $proxy = q{};
my $fileout = q{};
my $url = "http://www.baidu.com";
my $host = q{};
my $logpath = "/var/log/lighttpd/access.log";
my $moreInfo = q{};
my $ip = q{};
GetOptions(
'help|h'=>\$help,
'proxy=s'=>\$proxy,
'out=s'=>\$fileout,
'url=s'=>\$url,
'host|=s'=>\$host,
'logpath=s'=>\$logpath,
'vv'=>\$moreInfo,
'ip=s'=>\$ip,
);
if($help){
print <<__HELP__;
Notice: In order to run this script you must have those modules installed
cpan -i LWP::UserAgent
cpan -i Getopt::Long
cpan -i Term::ANSIColor
cpan -i LWP::Protocol::https
cpan -i LWP::Protocol::socks
cpan -i URI::Split
######## GET Basic Information about the proxy ####################
1. is Live or not
2. the connection spend time
Usage: $0 -proxy http://xxx.xxx.xxx.:7808 [-out xxx] [-url http://www.baidu.com] or
$0 -proxy proxyfile [-out xxx] [-url http://www.baidu.com]
-proxy: Specify the proxy,it can ben a proxy string like "http://23.228.65.132:7808" or proxy list filename
-out: Specify the filename to storage good proxy
-url: Specify the url for test proxy connection time, default is http://www.baidu.com
####### GET More Information about the proxy ######################
1. is Live or not
2. the connection spend time
3. proxy Anonymous degree
(1)High Anonymity Proxy :No VIA , NO X_Forwarded_For headers
(2)Anonymity Proxy: hide your ip
(3)Transparent Proxy: have proxy server info in via header and your ip in x_forwarded_for headers
Preconditions:
1. Install Web server: Lighttpd or Apache or Nginx or tomcat or IIS
2. usable domain/Internet IP with the web server
Usage: $0 -vv -proxy http://xxx.xxx.xxx.:7808 [-out xxx] -url http://YOUR DOMAIN -ip xxx.xxx.xxx [-host YourDomain] [-logpath Your Web Server Log path]
$0 -vv -proxy proxyfile [-out xxx] -url http://YOUR DOMAIN -ip xxx.xxx.xxx [-host YourDomain] [-logpath Your Web Server Log path]
-vv: Specify get proxy Anonymous degree
-url: Specify the domain bind your web server e.g. http://www.tanjiti.com
-host: Specify the hostname for Host header , default value is split from url
-logpath: Specify the web server log path e.g. /var/log/lighttpd/access.log for default lighttpd log path
-ip: Specify the server ip
-h: For more help
__HELP__
exit 0;
}
chomp $fileout;
chomp $proxy;
chomp $fileout;
chomp $url;
chomp $host;
chomp $logpath;
chomp $ip;
croak "You need to specify the proxy(proxyfile) \n Please run --help for more informations \n" if $proxy eq q{};
croak "$logpath is not readable " unless -r $logpath;
croak "You need to specify the url and the server ip \n Please run --help for more informations \n" if $moreInfo and ($url eq "http://www.baidu.com" or $ip eq q{});
##################################################################
## readFromFile(): storage the file contents into an array ##
## parameter: $filename ##
## return: @contents(array) ##
##################################################################
sub readFromFile{
my $filename = shift;
open my ($FH), "<", $filename or die "cannot open $filename for reading: $! \n";
my @contents = ();
while(<$FH>){
chomp $_;
push @contents, $_;
}
close $FH;
return @contents;
}
##################################################################
## writeToFile(): write the array datas into the specify file
## parameters: $content_ref, $filename
## return: no return
#################################################################
sub writeToFile{
my ($content_ref, $filename)= @_;
my @contents = @$content_ref;
open my ($FH), ">", $filename or die "cannot open $filename for writing: $! \n";
select $FH;
foreach (@contents){
say $_;
}
select STDOUT;
close $FH;
}
#################################################################
## isProxyLive: determine the proxy is live or not
## parameters: $proxy,$url,$host
## return: $isLive(boolean), $timeCost(number)
#################################################################
sub isProxyLive{
my ($proxy,$url,$host) = @_;
my ($scheme,$auth,$path,$query,$frag) = uri_split($url);
$host = $auth if defined $auth and $host eq q{};
my $ua = LWP::UserAgent->new;
$ua->agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:12.0) Gecko/20100101 Firefox/12.0");
$ua->timeout(10);
$ua->default_headers->push_header('Accept'=>'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8');
$ua->default_headers->push_header('Accept-Encoding'=>'gzip,deflate,sdch');
$ua->default_headers->push_header('Host' => $host);
$ua->default_headers->push_header('Connection' => 'keep-alive');
$ua->proxy([qw/http https/] => $proxy);
$ua->show_progress(1);
my $start = time();
my $response = $ua->get($url);
my $end = time();
my $timeCost = $end - $start;
my $isLive = $response->code == 200 ? 1: 0;
return ($isLive,$timeCost);
}
#################################################################
## getProxyType: determine the proxy type
## parameters: $logpath, $ip
## return: $ProxyType string (High Anonymity,Anoymity,Transparent)
#################################################################
sub getProxyType{
my($logpath, $ip) = @_;
my $log = `tail -1 $logpath`;
my $ProxyType = q{};
if ($log =~ /"\s+?"([-0-9.,a-zA-Z]+)"\s+?"(.+)"$/){
my $x_forwarded_for = $1;
my $via = $2;
#$isExposeIP == -1 : hide your ip
#$isExposeIP != -1 : expose your ip
my $isExposeIP = index($x_forwarded_for, $ip) ;
if($via eq "-" and $x_forwarded_for eq "-"){
$ProxyType = "High Anonymity Proxy ! No VIA , NO X_Forwarded_For include ip list ";
}elsif($isExposeIP != -1){
$ProxyType = "Transparent Proxy ! $via ";
}else{
$ProxyType = "Anonymity Proxy ! Hide your real IP !";
}
}
return $ProxyType;
}
my @proxys = -r $proxy ? readFromFile($proxy) : $proxy;
#@ok_proxys: storage the live proxy
my @ok_proxys = ();
foreach (@proxys){
my ($isLive, $timeCost) = isProxyLive($_,$url,$host);
if ($isLive){
my $ProxyDegree = getProxyType($logpath,$ip) if $moreInfo ne q{};
push @ok_proxys, $_ if $isLive;
my $proxyInfo = $moreInfo ? "[Proxy] $_ [TIME]: $timeCost [ProxyInfo]: $ProxyDegree" : "[Proxy] $_ [TIME]: $timeCost";
say BOLD RED $proxyInfo;
}else{
say BOLD YELLOW "Proxy: $_ NOT OK";
}
}
if (-r $proxy or $fileout){
$fileout = $proxy."_out" unless $fileout;
writeToFile(\@ok_proxys,$fileout);
}