Skip to content

Commit

Permalink
enhanced network scanner
Browse files Browse the repository at this point in the history 
- textarea input field can now be used to paste in a large list of hosts
- /31er subnet is possible (only one host)
- auto-detect subdomains for ftp and www subdomains
  • Loading branch information
@Orbiter
Orbiter committed Jul 8, 2013
1 parent d8354a3 commit 9f0cc9b
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 59 deletions.
13 changes: 6 additions & 7 deletions htroot/CrawlStartScanner_p.html
View file
Expand Up @@ -41,20 +41,19 @@ <h2>Network Scanner</h2>
</legend>
<dl>
<dt>Scan Range</dt>
<dd>
<dd align="top">
<input type="radio" name="source" id="sourcehost" value="hosts"#(intranet.checked)# checked="checked"::#(/intranet.checked)# />Scan sub-range with given host
<input type="text" name="scanhosts" value="#[scanhosts]#" size="60" maxlength="400" /><br/><br/>

<textarea name="scanhosts" id="scanhosts" value="#[scanhosts]#" cols="64" rows="3" size="41"></textarea><br/><br/>
<input type="radio" name="source" id="sourcenet" value="intranet"#(intranet.checked)#:: checked="checked"#(/intranet.checked)# />Full Intranet Scan: #[intranethosts]#<br/>
#(intranetHint)#::<div class="info">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Do not use intranet scan results, you are not in an intranet environment!</div>#(/intranetHint)#
</dd>
<dt>Time-Out</dt>
<dt>Subnet</dt>
<dd>
<input type="text" name="timeout" value ="100" size="4"/> ms
<input type="radio" name="subnet" value="31"/>/31 (only the given host(s)) <input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
</dd>
<dt>Subnet</dt>
<dt>Time-Out</dt>
<dd>
<input type="radio" name="subnet" value="24" checked="checked"/>/24 (254 addresses) <input type="radio" name="subnet" value="20"/>/20 (4064 addresses) <input type="radio" name="subnet" value="16"/>/16 (65024 adresses)
<input type="text" name="timeout" value ="500" size="4"/> ms
</dd>
<dt>Scan Cache</dt>
<dd>
Expand Down
70 changes: 57 additions & 13 deletions htroot/CrawlStartScanner_p.java
View file
Expand Up @@ -18,27 +18,33 @@
* If not, see <http://www.gnu.org/licenses/>.
*/

import java.io.IOException;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.util.ConcurrentModificationException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;

import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.Scanner;
import net.yacy.cora.protocol.Scanner.Access;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.data.WorkTables;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;

Expand Down Expand Up @@ -66,11 +72,20 @@ public static serverObjects respond(
timeout = post == null ? timeout : post.getInt("timeout", timeout);

// make a scanhosts entry
String hosts = post == null ? "" : post.get("scanhosts", "");
String hostt = post == null ? "" : post.get("scanhosts", "").trim();
boolean listall = false;
if (hostt.equals("*")) {
hostt = "";
listall = true;
}
String[] hosts0 = hostt.indexOf('\n') > 0 || hostt.indexOf('\r') > 0 ? hostt.split("[\\r\\n]+") : hostt.split(Pattern.quote(","));
Set<String> hostSet = new LinkedHashSet<String>();
for (String s: hosts0) if (s != null && s.length() > 0) hostSet.add(s);

final Set<InetAddress> ips = Domains.myIntranetIPs();
prop.put("intranethosts", ips.toString());
prop.put("intranetHint", sb.isIntranetMode() ? 0 : 1);
if ( hosts.isEmpty() ) {
if ( hostSet.isEmpty() ) {
InetAddress ip;
if ( sb.isIntranetMode() ) {
if ( !ips.isEmpty() ) {
Expand All @@ -85,11 +100,23 @@ public static serverObjects respond(
}
}
if ( ip != null ) {
hosts = ip.getHostAddress();
hostSet.add(ip.getHostAddress());
}
}
prop.put("scanhosts", hosts);
String hos = ""; for (String s: hostSet) hos += s + "\n";
prop.put("scanhosts", hos.trim());

if (listall) {
// get a list of all hosts in the index
ReversibleScoreMap<String> hostscore = null;
try {
hostscore = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_TERM, 1000, CollectionSchema.host_s.getSolrFieldName()).get(CollectionSchema.host_s.getSolrFieldName());
} catch (IOException e) {}
if (hostscore != null) {
for (String s: hostscore) hostSet.add(s);
}
}

// parse post requests
if ( post != null ) {
int repeat_time = 0;
Expand All @@ -105,31 +132,48 @@ public static serverObjects respond(

// scan a range of ips
if (post.containsKey("scan")) {

boolean scanftp = "on".equals(post.get("scanftp", ""));
boolean scanhttp = "on".equals(post.get("scanhttp", ""));
boolean scanhttps = "on".equals(post.get("scanhttps", ""));
boolean scansmb = "on".equals(post.get("scansmb", ""));

final Set<InetAddress> scanbase = new HashSet<InetAddress>();

// select host base to scan
if ("hosts".equals(post.get("source", ""))) {
for (String host: hosts.split(",")) {
for (String host: hostSet) {
if (host.startsWith("http://")) host = host.substring(7);
if (host.startsWith("https://")) host = host.substring(8);
if (host.startsWith("ftp://")) host = host.substring(6);
if (host.startsWith("smb://")) host = host.substring(6);
final int p = host.indexOf('/', 0);
if (p >= 0) host = host.substring(0, p);
if (host.length() > 0) scanbase.add(Domains.dnsResolve(host));
InetAddress ip;
if (host.length() > 0) {
ip = Domains.dnsResolve(host); if (ip != null) scanbase.add(ip);
if (scanftp && !hostSet.contains("ftp." + host)) {
ip = Domains.dnsResolve("ftp." + host);
if (ip != null) scanbase.add(ip);
}
if ((scanhttp || scanhttps) && !hostSet.contains("www." + host)) {
ip = Domains.dnsResolve("www." + host);
if (ip != null) scanbase.add(ip);
}
}
}
}
if ("intranet".equals(post.get("source", ""))) {
scanbase.addAll(Domains.myIntranetIPs());
}

// start a scanner
final Scanner scanner = new Scanner(scanbase, CONCURRENT_RUNNER, timeout);
List<InetAddress> addresses = scanner.genlist(subnet);
if ("on".equals(post.get("scanftp", ""))) scanner.addFTP(addresses);
if ("on".equals(post.get("scanhttp", ""))) scanner.addHTTP(addresses);
if ("on".equals(post.get("scanhttps", ""))) scanner.addHTTPS(addresses);
if ("on".equals(post.get("scansmb", ""))) scanner.addSMB(addresses);
final Scanner scanner = new Scanner(CONCURRENT_RUNNER, timeout);
List<InetAddress> addresses = Scanner.genlist(scanbase, subnet);
if (scanftp) scanner.addFTP(addresses);
if (scanhttp) scanner.addHTTP(addresses);
if (scanhttps) scanner.addHTTPS(addresses);
if (scansmb) scanner.addSMB(addresses);
scanner.start();
scanner.terminate();
if ("on".equals(post.get("accumulatescancache", "")) && !"scheduler".equals(post.get("rescan", ""))) {
Expand Down Expand Up @@ -183,7 +227,7 @@ public static serverObjects respond(
post,
"CrawlStartScanner_p.html",
WorkTables.TABLE_API_TYPE_CRAWLER,
"network scanner for hosts: " + hosts,
"network scanner for hosts: " + hostSet.toString(),
repeat_time,
repeat_unit.substring(3));
}
Expand Down
75 changes: 36 additions & 39 deletions source/net/yacy/cora/protocol/Scanner.java
View file
Expand Up @@ -29,13 +29,12 @@
import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
Expand Down Expand Up @@ -121,9 +120,6 @@ public boolean equals(final Object o) {
}

private final static Map<Service, Access> scancache = new ConcurrentHashMap<Service, Access>();
//private static long scancacheUpdateTime = 0;
//private static long scancacheValidUntilTime = Long.MAX_VALUE;
private static Set<InetAddress> scancacheScanrange = new HashSet<InetAddress>();

public static int scancacheSize() {
return scancache.size();
Expand All @@ -132,9 +128,6 @@ public static int scancacheSize() {
public static void scancacheReplace(final Scanner newScanner) {
scancache.clear();
scancache.putAll(newScanner.services());
//scancacheUpdateTime = System.currentTimeMillis();
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
scancacheScanrange = newScanner.scanrange;
}

public static void scancacheExtend(final Scanner newScanner) {
Expand All @@ -145,9 +138,6 @@ public static void scancacheExtend(final Scanner newScanner) {
if (entry.getValue() != Access.granted) i.remove();
}
scancache.putAll(newScanner.services());
//scancacheUpdateTime = System.currentTimeMillis();
//scancacheValidUntilTime = validTime == Long.MAX_VALUE ? Long.MAX_VALUE : scancacheUpdateTime + validTime;
scancacheScanrange = newScanner.scanrange;
}

public static Iterator<Map.Entry<Service, Scanner.Access>> scancacheEntries() {
Expand All @@ -163,18 +153,17 @@ public static Iterator<Map.Entry<Service, Scanner.Access>> scancacheEntries() {
*/
public static boolean acceptURL(final MultiProtocolURI url) {
// if the scan range is empty, then all urls are accepted
if (scancacheScanrange == null || scancacheScanrange.isEmpty()) return true;
if (scancache == null || scancache.isEmpty()) return true;

//if (System.currentTimeMillis() > scancacheValidUntilTime) return true;
final InetAddress a = url.getInetAddress(); // try to avoid that!
if (a == null) return true;
final InetAddress n = normalize(a);
if (!scancacheScanrange.contains(n)) return true;
final Access access = scancache.get(new Service(url.getProtocol(), a));
if (access == null) return false;
return access == Access.granted;
}


/*
private static InetAddress normalize(final InetAddress a) {
if (a == null) return null;
final byte[] b = a.getAddress();
Expand All @@ -186,28 +175,22 @@ private static InetAddress normalize(final InetAddress a) {
return a;
}
}

*/

private final int runnerCount;
private final Set<InetAddress> scanrange;
private final BlockingQueue<Service> scanqueue;
private final Map<Service, Access> services;
private final Map<Runner, Object> runner;
private final int timeout;

public Scanner(final Set<InetAddress> scanrange, final int concurrentRunner, final int timeout) {
public Scanner(final int concurrentRunner, final int timeout) {
this.runnerCount = concurrentRunner;
this.scanrange = new HashSet<InetAddress>();
for (final InetAddress a: scanrange) this.scanrange.add(normalize(a));
this.scanqueue = new LinkedBlockingQueue<Service>();
this.services = Collections.synchronizedMap(new HashMap<Service, Access>());
this.runner = new ConcurrentHashMap<Runner, Object>();
this.timeout = timeout;
}

public Scanner(final int concurrentRunner, final int timeout) {
this(Domains.myIntranetIPs(), concurrentRunner, timeout);
}

@Override
public void run() {
Service uri;
Expand Down Expand Up @@ -332,23 +315,37 @@ private void addProtocol(final Protocol protocol, final List<InetAddress> addres
* @param subnet the subnet: 24 will generate 254 addresses, 16 will generate 256 * 254; must be >= 16 and <= 24
* @return
*/
public final List<InetAddress> genlist(final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(10);
for (final InetAddress i: this.scanrange) {
int ul = subnet >= 24 ? i.getAddress()[2] : (1 << (24 - subnet)) - 1;
for (int br = subnet >= 24 ? i.getAddress()[2] : 0; br <= ul; br++) {
for (int j = 1; j < 255; j++) {
final byte[] address = i.getAddress();
address[2] = (byte) br;
address[3] = (byte) j;
try {
c.add(InetAddress.getByAddress(address));
} catch (final UnknownHostException e) {
public static final List<InetAddress> genlist(Collection<InetAddress> base, final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
for (final InetAddress i: base) {
genlist(c, i, subnet);
}
return c;
}
public static final List<InetAddress> genlist(InetAddress base, final int subnet) {
final ArrayList<InetAddress> c = new ArrayList<InetAddress>(1);
genlist(c, base, subnet);
return c;
}
private static final void genlist(ArrayList<InetAddress> c, InetAddress base, final int subnet) {
if (subnet == 31) {
try {
c.add(InetAddress.getByAddress(base.getAddress()));
} catch (UnknownHostException e) {}
} else {
int ul = subnet >= 24 ? base.getAddress()[2] : (1 << (24 - subnet)) - 1;
for (int br = subnet >= 24 ? base.getAddress()[2] : 0; br <= ul; br++) {
for (int j = 1; j < 255; j++) {
final byte[] address = base.getAddress();
address[2] = (byte) br;
address[3] = (byte) j;
try {
c.add(InetAddress.getByAddress(address));
} catch (final UnknownHostException e) {
}
}
}
}
}
return c;
}

public Map<Service, Access> services() {
Expand All @@ -365,7 +362,7 @@ public static byte[] inIndex(final Map<byte[], String> commentCache, final Strin
public static void main(final String[] args) {
//try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {}
final Scanner scanner = new Scanner(100, 10);
List<InetAddress> addresses = scanner.genlist(20);
List<InetAddress> addresses = genlist(Domains.myIntranetIPs(), 20);
scanner.addFTP(addresses);
scanner.addHTTP(addresses);
scanner.addHTTPS(addresses);
Expand Down

0 comments on commit 9f0cc9b

Please sign in to comment.