Skip to content

Commit

Permalink
use DigestURL in UrlProxyServlet as parameter to pass requested url to
Browse files Browse the repository at this point in the history 
handler.
UrlProxyServlet splits url in parts to pass it on as parameter and 
HeaderFramework constructs a url from param parts. This is obsolete if
already created url is used (makes HeaderFramework.getRequestURL obsolete
= removed)
  • Loading branch information
reger committed Nov 20, 2016
1 parent 3630fcc commit 4eeb448
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 116 deletions.
36 changes: 1 addition & 35 deletions source/net/yacy/cora/protocol/HeaderFramework.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,10 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
Expand All @@ -40,10 +38,8 @@

import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.NumberTools;


/**
Expand Down Expand Up @@ -200,14 +196,14 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public static final String CONNECTION_PROP_METHOD = "METHOD";
public static final String CONNECTION_PROP_PATH = "PATH";
public static final String CONNECTION_PROP_EXT = "EXT";
public static final String CONNECTION_PROP_URL = "URL";
public static final String CONNECTION_PROP_ARGS = "ARGS";
public static final String CONNECTION_PROP_CLIENTIP = "CLIENTIP";
public static final String CONNECTION_PROP_PERSISTENT = "PERSISTENT";
public static final String CONNECTION_PROP_REQUEST_START = "REQUEST_START";
public static final String CONNECTION_PROP_REQUEST_END = "REQUEST_END";

/* PROPERTIES: Client -> Proxy */
public static final String CONNECTION_PROP_DIGESTURL = "URL"; // value DigestURL object
public static final String CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST = "CLIENT_HTTPSERVLETREQUEST";

/* PROPERTIES: Proxy -> Client */
Expand Down Expand Up @@ -565,36 +561,6 @@ public void toHeaderString(
theHeader.append("\r\n");
}

/**
* Generate a url from Header properties
* @param conProp containing host, path, query and protocol (defaults to http if missing)
* @return url
* @throws MalformedURLException
*/
public static DigestURL getRequestURL(final HashMap<String, Object> conProp) throws MalformedURLException {
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
String protocol = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PROTOCOL);
if (protocol == null) protocol = "http";
//String ip = conProp.getProperty(httpHeader.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer

int port, pos;
if ((pos = host.lastIndexOf(':')) < 0) {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
} else {
if (pos > host.indexOf(']')) { // check for ipv6
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = Domains.stripToPort(protocol + "://" + host); // use stripToPort to get default ports
}
}

final DigestURL url = new DigestURL(protocol, host, port, (args == null) ? path : path + "?" + args);
return url;
}

/**
* Reading http headers from a reader class and building up a httpHeader object
* @param reader the {@link BufferedReader} that is used to read the http header lines
Expand Down
12 changes: 4 additions & 8 deletions source/net/yacy/http/servlets/UrlProxyServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
Expand Down Expand Up @@ -122,17 +121,17 @@ public void service (ServletRequest req, ServletResponse res) throws ServletExce
return;
}
// 2 - get target url
URL proxyurl = null;
DigestURL proxyurl = null;
final String strUrl = request.getParameter("url");
if (strUrl == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
return;
}

try {
proxyurl = new URL(strUrl);
proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
}

if (proxyurl == null) {
Expand All @@ -151,11 +150,8 @@ public void service (ServletRequest req, ServletResponse res) throws ServletExce

final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_METHOD, request.getMethod()); // only needed for HTTPDeamon errormsg in case of blacklisted url
if (proxyurl.getQuery() != null) prop.put(HeaderFramework.CONNECTION_PROP_ARGS, proxyurl.getQuery());
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

Expand Down
12 changes: 4 additions & 8 deletions source/net/yacy/http/servlets/YaCyProxyServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import java.io.InputStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
Expand Down Expand Up @@ -91,7 +90,7 @@ public void service (ServletRequest req, ServletResponse res) throws ServletExce
response.sendError(HttpServletResponse.SC_GATEWAY_TIMEOUT); // Need better test that isInitial
return;
}
URL proxyurl = null;
DigestURL proxyurl = null;
String strARGS = request.getQueryString();
if (strARGS == null) {
response.sendError(HttpServletResponse.SC_NOT_FOUND,"url parameter missing");
Expand All @@ -102,10 +101,9 @@ public void service (ServletRequest req, ServletResponse res) throws ServletExce
final String strUrl = strARGS.substring(4); // strip "url="

try {
proxyurl = new URL(strUrl);
proxyurl = new DigestURL(strUrl);
} catch (final MalformedURLException e) {
proxyurl = new URL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));

proxyurl = new DigestURL(URLDecoder.decode(strUrl, StandardCharsets.UTF_8.name()));
}
}
if (proxyurl == null) {
Expand All @@ -123,9 +121,7 @@ public void service (ServletRequest req, ServletResponse res) throws ServletExce

final HashMap<String, Object> prop = new HashMap<String, Object>();
prop.put(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_1_1);
prop.put(HeaderFramework.CONNECTION_PROP_PROTOCOL, proxyurl.getProtocol());
prop.put(HeaderFramework.CONNECTION_PROP_HOST, hostwithport);
prop.put(HeaderFramework.CONNECTION_PROP_PATH, proxyurl.getPath().replaceAll(" ", "%20"));
prop.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, proxyurl);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, Domains.LOCALHOST);
prop.put(HeaderFramework.CONNECTION_PROP_CLIENT_HTTPSERVLETREQUEST, request);

Expand Down
79 changes: 31 additions & 48 deletions source/net/yacy/server/http/HTTPDProxyHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,42 +277,32 @@ public static void doGet(final HashMap<String, Object> conProp, final RequestHea

final String ip = (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the ip from the connecting peer

DigestURL url = null;
try {
url = HeaderFramework.getRequestURL(conProp);
if (log.isFine()) log.fine(reqID +" GET "+ url);
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);

//redirector
if (redirectorEnabled){
synchronized(redirectorProcess){
redirectorWriter.println(url.toNormalform(true));
redirectorWriter.flush();
}
final String newUrl = redirectorReader.readLine();
if (!newUrl.equals("")) {
try {
url = new DigestURL(newUrl);
} catch(final MalformedURLException e){}//just keep the old one
}
if (log.isFinest()) log.finest(reqID +" using redirector to "+ url);
conProp.put(HeaderFramework.CONNECTION_PROP_HOST, url.getHost()+":"+url.getPort());
conProp.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
requestHeader.put(HeaderFramework.HOST, url.getHost()+":"+url.getPort());
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
if (log.isFine()) log.fine(reqID +" GET "+ url.toString());
if (log.isFinest()) log.finest(reqID +" header: "+ requestHeader);

//redirector
if (redirectorEnabled) {
synchronized (redirectorProcess) {
redirectorWriter.println(url.toNormalform(true));
redirectorWriter.flush();
}
} catch (final MalformedURLException e) {
// get header info for error logging
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/'
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given
final String errorMsg = "ERROR: internal error with url generation: host=" +
host + ", path=" + path + ", args=" + args;
log.severe(errorMsg);
HTTPDemon.sendRespondError(conProp,countedRespond,4,501,null,errorMsg,e);
return;
final String newUrl = redirectorReader.readLine();
if (!newUrl.equals("")) {
try {
url = new DigestURL(newUrl);
} catch (final MalformedURLException e) {
}//just keep the old one
}
if (log.isFinest()) {
log.finest(reqID + " using redirector to " + url);
}
conProp.put(HeaderFramework.CONNECTION_PROP_DIGESTURL, url);
requestHeader.put(HeaderFramework.HOST, url.getHost() + ":" + url.getPort());
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, url.getPath());
}


// check the blacklist
// blacklist idea inspired by [AS]:
// respond a 404 for all AGIS ("all you get is shit") servers
Expand Down Expand Up @@ -1049,14 +1039,13 @@ private static serverObjects unknownHostHandling(final HashMap<String, Object> c
}));

// getting some connection properties
String orgHostPort = "80";
String orgHostName = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
DigestURL orgurl = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
int orgHostPort = orgurl.getPort();
String orgHostName = orgurl.getHost();
if (orgHostName == null) orgHostName = "unknown";
orgHostName = orgHostName.toLowerCase();
orgHostPort = Integer.toString(Domains.stripToPort(orgHostName));
orgHostName = Domains.stripToHostName(orgHostName);
String orgHostPath = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); if (orgHostArgs == null) orgHostArgs = "";
String orgHostPath = orgurl.getPath(); if (orgHostPath == null) orgHostPath = "";
String orgHostArgs = orgurl.getSearchpart();; if (orgHostArgs == null) orgHostArgs = "";
if (orgHostArgs.length() > 0) orgHostArgs = "?" + orgHostArgs;
detailedErrorMsgMap.put("hostName", orgHostName);

Expand Down Expand Up @@ -1183,23 +1172,17 @@ private final static synchronized void logProxyAccess(final HashMap<String, Obje
logMessage.append(' ');

// URL
final String requestURL = (String) conProp.get(HeaderFramework.CONNECTION_PROP_URL);
final String requestArgs = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
logMessage.append(requestURL);
if (requestArgs != null) {
logMessage.append("?")
.append(requestArgs);
}
final DigestURL requestURL = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);
logMessage.append(requestURL.toString());
logMessage.append(' ');

// Rfc931
logMessage.append("-");
logMessage.append(' ');

// Peerstatus/Peerhost
final String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
logMessage.append("DIRECT/");
logMessage.append(host);
logMessage.append(requestURL.getHost());
logMessage.append(' ');

// Type
Expand Down
21 changes: 4 additions & 17 deletions source/net/yacy/server/http/HTTPDemon.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,12 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
Expand Down Expand Up @@ -138,29 +136,18 @@ private static final void sendRespondError(
}

// generating the desired request url
String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST);
String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); if (path == null) path = "/";
final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS);
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);

final int port = Domains.stripToPort(host);
host = Domains.stripToHostName(host);

String urlString;
try {
urlString = (new DigestURL((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString();
} catch (final MalformedURLException e) {
urlString = "invalid URL";
}
final String method = (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD);
DigestURL url = (DigestURL) conProp.get(HeaderFramework.CONNECTION_PROP_DIGESTURL);

// set rewrite values
final serverObjects tp = new serverObjects();

tp.put("peerName", (switchboard.peers == null) ? "" : switchboard.peers.myName());
tp.put("errorMessageType", Integer.toString(errorcase));
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", (String) conProp.get(HeaderFramework.CONNECTION_PROP_METHOD));
tp.put("requestURL", urlString);
tp.put("requestMethod", method);
tp.put("requestURL", url.toString());

switch (errorcase) {
case ERRORCASE_FILE:
Expand Down

0 comments on commit 4eeb448

Please sign in to comment.