Experimental SOCKS4a proxy support added, checkstyle changed, v4.2.5

checkstyle.xml

@@ -99,9 +99,15 @@
 
         <!-- Checks for Javadoc comments.                     -->
         <!-- See http://checkstyle.sf.net/config_javadoc.html -->
-        <module name="JavadocMethod"/>
-        <module name="JavadocType"/>
-        <module name="JavadocVariable"/>
+        <module name="JavadocMethod">
+            <property name="scope" value="public"/>
+        </module>
+        <module name="JavadocType">
+            <property name="scope" value="public"/>
+        </module>
+        <module name="JavadocVariable">
+            <property name="scope" value="public"/>
+        </module>
         <module name="JavadocStyle"/>
 
         <!-- Checks for Naming Conventions.                  -->
@@ -176,7 +182,9 @@
 
         <!-- Checks for class design                         -->
         <!-- See http://checkstyle.sf.net/config_design.html -->
-        <module name="DesignForExtension"/>
+        <module name="DesignForExtension">
+            <property name="ignoredAnnotations" value="Override, Test"/>
+        </module>
         <module name="FinalClass"/>
         <module name="HideUtilityClassConstructor"/>
         <module name="InterfaceIsType"/>
@@ -185,7 +193,7 @@
         <!-- Miscellaneous other checks.                   -->
         <!-- See http://checkstyle.sf.net/config_misc.html -->
         <module name="ArrayTypeStyle"/>
-        <module name="FinalParameters"/>
+        <!-- <module name="FinalParameters"/> -->
         <module name="TodoComment"/>
         <module name="UpperEll"/>
 

pom.xml

@@ -10,7 +10,7 @@
 
     <groupId>ai.preferred</groupId>
     <artifactId>venom</artifactId>
-    <version>4.2.4</version>
+    <version>4.2.5</version>
     <packaging>jar</packaging>
 
     <name>${project.groupId}:${project.artifactId}</name>

src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java

@@ -21,6 +21,9 @@
 import ai.preferred.venom.request.HttpFetcherRequest;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.Response;
+import ai.preferred.venom.socks.SocksConnectingIOReactor;
+import ai.preferred.venom.socks.SocksHttpRoutePlanner;
+import ai.preferred.venom.socks.SocksIOSessionStrategy;
 import ai.preferred.venom.storage.FileManager;
 import ai.preferred.venom.uagent.DefaultUserAgent;
 import ai.preferred.venom.uagent.UserAgent;
@@ -42,11 +45,20 @@
 import org.apache.http.client.utils.URIUtils;
 import org.apache.http.concurrent.BasicFuture;
 import org.apache.http.concurrent.FutureCallback;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
 import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.conn.DefaultRoutePlanner;
+import org.apache.http.impl.conn.DefaultSchemePortResolver;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
+import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
 import org.apache.http.impl.nio.reactor.IOReactorConfig;
 import org.apache.http.nio.client.methods.HttpAsyncMethods;
+import org.apache.http.nio.conn.NoopIOSessionStrategy;
+import org.apache.http.nio.conn.SchemeIOSessionStrategy;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.apache.http.nio.reactor.IOReactorException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -186,13 +198,33 @@ private AsyncFetcher(final Builder builder) {
         .build();
 
     final HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create()
-        .setDefaultIOReactorConfig(reactorConfig)
-        .setThreadFactory(builder.threadFactory)
         .setMaxConnPerRoute(builder.maxRouteConnections)
         .setMaxConnTotal(builder.maxConnections)
         .setSSLContext(builder.sslContext)
         .setRedirectStrategy(builder.redirectStrategy);
 
+    if (builder.enableSocksProxy) {
+      final PoolingNHttpClientConnectionManager connectionManager;
+      try {
+        final SSLIOSessionStrategy sslioSessionStrategy = SSLIOSessionStrategy.getDefaultStrategy();
+        final Registry<SchemeIOSessionStrategy> reg = RegistryBuilder.<SchemeIOSessionStrategy>create()
+            .register("socks", new SocksIOSessionStrategy(sslioSessionStrategy))
+            .register("http", NoopIOSessionStrategy.INSTANCE)
+            .register("https", sslioSessionStrategy)
+            .build();
+
+        final SocksConnectingIOReactor reactor = new SocksConnectingIOReactor(reactorConfig, builder.threadFactory);
+        connectionManager = new PoolingNHttpClientConnectionManager(reactor, reg);
+        clientBuilder.setConnectionManager(connectionManager)
+            .setRoutePlanner(new SocksHttpRoutePlanner(new DefaultRoutePlanner(DefaultSchemePortResolver.INSTANCE)));
+      } catch (IOReactorException e) {
+        LOGGER.error("Disabling SOCKS protocol", e);
+        clientBuilder.setDefaultIOReactorConfig(reactorConfig).setThreadFactory(builder.threadFactory);
+      }
+    } else {
+      clientBuilder.setDefaultIOReactorConfig(reactorConfig).setThreadFactory(builder.threadFactory);
+    }
+
     if (builder.maxConnections < builder.maxRouteConnections) {
       clientBuilder.setMaxConnTotal(builder.maxRouteConnections);
       LOGGER.info("Maximum total connections will be set to {}, to match maximum route connection.",
@@ -442,6 +474,8 @@ public static final class Builder {
      */
     private final List<Callback> callbacks;
 
+    private boolean enableSocksProxy;
+
     /**
      * Determines whether cookie storage is allowed.
      */
@@ -555,6 +589,17 @@ private Builder() {
       connectTimeout = -1;
       socketTimeout = -1;
       compressed = true;
+      enableSocksProxy = false;
+    }
+
+    /**
+     * Enables SOCKS protocol for proxies (socks://). Experimental.
+     *
+     * @return this
+     */
+    public Builder enableSocksProxy() {
+      enableSocksProxy = true;
+      return this;
     }
 
     /**

src/main/java/ai/preferred/venom/socks/SocksConnectingIOReactor.java

@@ -0,0 +1,53 @@
+package ai.preferred.venom.socks;
+
+import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
+import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.apache.http.nio.reactor.IOEventDispatch;
+import org.apache.http.nio.reactor.IOReactorException;
+
+import java.io.InterruptedIOException;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ * This IOReactor makes sure that the supplied {@link IOEventDispatch} is decorated with {@link SocksIOEventDispatch}.
+ */
+public class SocksConnectingIOReactor extends DefaultConnectingIOReactor {
+
+  /**
+   * Creates an instance of SocksConnectingIOReactor with the given configuration.
+   *
+   * @param config        I/O reactor configuration.
+   * @param threadFactory the factory to create threads.
+   *                      Can be {@code null}.
+   * @throws IOReactorException in case if a non-recoverable I/O error.
+   */
+  public SocksConnectingIOReactor(IOReactorConfig config, ThreadFactory threadFactory) throws IOReactorException {
+    super(config, threadFactory);
+  }
+
+  /**
+   * Creates an instance of SocksConnectingIOReactor with the given configuration.
+   *
+   * @param config I/O reactor configuration.
+   *               Can be {@code null}.
+   * @throws IOReactorException in case if a non-recoverable I/O error.
+   */
+  public SocksConnectingIOReactor(IOReactorConfig config) throws IOReactorException {
+    super(config);
+  }
+
+  /**
+   * Creates an instance of SocksConnectingIOReactor with default configuration.
+   *
+   * @throws IOReactorException in case if a non-recoverable I/O error.
+   */
+  public SocksConnectingIOReactor() throws IOReactorException {
+    super();
+  }
+
+  @Override
+  public void execute(final IOEventDispatch eventDispatch) throws InterruptedIOException, IOReactorException {
+    super.execute(new SocksIOEventDispatch(eventDispatch));
+  }
+
+}

src/main/java/ai/preferred/venom/socks/SocksHttpRoutePlanner.java

@@ -0,0 +1,40 @@
+package ai.preferred.venom.socks;
+
+import com.google.common.annotations.Beta;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.routing.HttpRoutePlanner;
+import org.apache.http.protocol.HttpContext;
+
+/**
+ * This route planners ensures that the connection to https server via socks proxy works. It prevents http client from
+ * tunnelling the IO session twice ({@link SocksIOSessionStrategy} upgrades {@link SocksIOSession} to
+ * {@link org.apache.http.nio.reactor.ssl.SSLIOSession} when necessary).
+ */
+@Beta
+public class SocksHttpRoutePlanner implements HttpRoutePlanner {
+
+  private final HttpRoutePlanner rp;
+
+  /**
+   * Decorates {@link HttpRoutePlanner}.
+   *
+   * @param rp decorated route planner
+   */
+  public SocksHttpRoutePlanner(final HttpRoutePlanner rp) {
+    this.rp = rp;
+  }
+
+  @Override
+  public HttpRoute determineRoute(HttpHost host, HttpRequest request, HttpContext context) throws HttpException {
+    final HttpRoute route = rp.determineRoute(host, request, context);
+    final boolean secure = "https".equalsIgnoreCase(route.getTargetHost().getSchemeName());
+    if (secure && route.getProxyHost() != null && "socks".equalsIgnoreCase(route.getProxyHost().getSchemeName())) {
+      return new HttpRoute(route.getTargetHost(), route.getLocalAddress(), route.getProxyHost(), false);
+    }
+    return route;
+  }
+
+}

src/main/java/ai/preferred/venom/socks/SocksIOEventDispatch.java

@@ -0,0 +1,118 @@
+package ai.preferred.venom.socks;
+
+import org.apache.http.nio.NHttpClientConnection;
+import org.apache.http.nio.NHttpClientEventHandler;
+import org.apache.http.nio.NHttpConnection;
+import org.apache.http.nio.protocol.HttpAsyncRequestExecutor;
+import org.apache.http.nio.reactor.IOEventDispatch;
+import org.apache.http.nio.reactor.IOSession;
+
+import java.io.IOException;
+
+/**
+ * This class wraps and handles IO dispatch related to {@link SocksIOSession}.
+ */
+public class SocksIOEventDispatch implements IOEventDispatch {
+
+  private final IOEventDispatch dispatch;
+
+  /**
+   * Decorates {@link IOEventDispatch}.
+   *
+   * @param dispatch delegated IO dispatch
+   */
+  public SocksIOEventDispatch(IOEventDispatch dispatch) {
+    this.dispatch = dispatch;
+  }
+
+  @Override
+  public void connected(IOSession session) {
+    dispatch.connected(session);
+  }
+
+  @Override
+  public void inputReady(IOSession session) {
+    try {
+      if (initializeSocksSession(session)) {
+        dispatch.inputReady(session);
+      }
+    } catch (RuntimeException e) {
+      session.shutdown();
+      throw e;
+    }
+  }
+
+  @Override
+  public void outputReady(IOSession session) {
+    try {
+      if (initializeSocksSession(session)) {
+        dispatch.outputReady(session);
+      }
+    } catch (RuntimeException e) {
+      session.shutdown();
+      throw e;
+    }
+  }
+
+  @Override
+  public void timeout(IOSession session) {
+    try {
+      dispatch.timeout(session);
+      final SocksIOSession socksIOSession = getSocksSession(session);
+      if (socksIOSession != null) {
+        socksIOSession.shutdown();
+      }
+    } catch (RuntimeException e) {
+      session.shutdown();
+      throw e;
+    }
+  }
+
+  @Override
+  public void disconnected(IOSession session) {
+    dispatch.disconnected(session);
+  }
+
+  private boolean initializeSocksSession(IOSession session) {
+    final SocksIOSession socksSession = getSocksSession(session);
+    if (socksSession != null) {
+      try {
+        try {
+          if (!socksSession.isInitialized()) {
+            return socksSession.initialize();
+          }
+        } catch (final IOException e) {
+          onException(socksSession, e);
+          throw new RuntimeException(e);
+        }
+      } catch (final RuntimeException e) {
+        socksSession.shutdown();
+        throw e;
+      }
+    }
+    return true;
+  }
+
+  private void onException(IOSession session, Exception ex) {
+    final NHttpClientConnection conn = getConnection(session);
+    if (conn != null) {
+      final NHttpClientEventHandler handler = getEventHandler(conn);
+      if (handler != null) {
+        handler.exception(conn, ex);
+      }
+    }
+  }
+
+  private SocksIOSession getSocksSession(IOSession session) {
+    return (SocksIOSession) session.getAttribute(SocksIOSession.SESSION_KEY);
+  }
+
+  private NHttpClientConnection getConnection(IOSession session) {
+    return (NHttpClientConnection) session.getAttribute(IOEventDispatch.CONNECTION_KEY);
+  }
+
+  private NHttpClientEventHandler getEventHandler(NHttpConnection conn) {
+    return (NHttpClientEventHandler) conn.getContext().getAttribute(HttpAsyncRequestExecutor.HTTP_HANDLER);
+  }
+
+}