Upgrade Netty 4.1 and AHC 2.1

Motivation:

Netty 4.1 is pretty stable and has the DNS codec that I have to fork
for AHC 2.0, which is quite some extra work.

Modifications:

* Upgrade Netty 4.1.8
* Upgrade AHC 2.1.0-alpha5
* Use Cookie API from Netty as it was dropped from AHC 2.1.
* rename `acceptAnyCertificate` config parameter into
`disableHttpsAlgorithm`

Results:

Gatling now uses Netty 4.1.

There are 2 FIXMEs left (te be resolved in Netty 4.1.9):
* in CookieJar, use constant defined in
netty/netty#6327
* in ExtendedDnsNameResolver, as AHC DNS codec fork is more up-to-date
than the one in Netty 4.1.8

gatling-core/src/main/resources/config-renamed.properties

@@ -13,3 +13,4 @@ gatling.core.timeOut.simulation=gatling.core.timeout.simulation
 gatling.http.ahc.httpsEnabledProtocols=gatling.http.ahc.sslEnabledProtocols
 gatling.http.ahc.httpsEnabledCipherSuites=gatling.http.ahc.sslEnabledCipherSuites
 gatling.http.ahc.allowPoolingConnections=gatling.http.ahc.keepAlive
+gatling.http.ahc.acceptAnyCertificate=gatling.http.ahc.disableHttpsAlgorithm

gatling-core/src/main/resources/gatling-defaults.conf

@@ -82,7 +82,7 @@ gatling {
       readTimeout = 60000                             # Timeout when a used connection stays idle
       maxRetry = 2                                    # Number of times that a request should be tried again
       requestTimeout = 60000                          # Timeout of the requests
-      acceptAnyCertificate = true                     # When set to true, doesn't validate SSL certificates
+      disableHttpsAlgorithm = true                    # When set to true, enable SSL algorythm on the SSLEngine
       httpClientCodecMaxChunkSize = 8192              # Maximum length of the content or each chunk
       sslEnabledProtocols = [TLSv1.2, TLSv1.1, TLSv1] # Array of enabled protocols for HTTPS, if empty use the JDK defaults
       sslEnabledCipherSuites = []                     # Array of enabled cipher suites for HTTPS, if empty use the AHC defaults

gatling-core/src/main/scala/io/gatling/core/ConfigKeys.scala

@@ -98,7 +98,7 @@ object ConfigKeys {
       val ReadTimeout = "gatling.http.ahc.readTimeout"
       val MaxRetry = "gatling.http.ahc.maxRetry"
       val RequestTimeout = "gatling.http.ahc.requestTimeout"
-      val AcceptAnyCertificate = "gatling.http.ahc.acceptAnyCertificate"
+      val DisableHttpsAlgorithm = "gatling.http.ahc.disableHttpsAlgorithm"
       val HttpClientCodecMaxChunkSize = "gatling.http.ahc.httpClientCodecMaxChunkSize"
       val SslEnabledProtocols = "gatling.http.ahc.sslEnabledProtocols"
       val SslEnabledCipherSuites = "gatling.http.ahc.sslEnabledCipherSuites"

gatling-core/src/main/scala/io/gatling/core/config/GatlingConfiguration.scala

@@ -178,13 +178,13 @@ object GatlingConfiguration extends StrictLogging {
           readTimeout = config.getInt(http.ahc.ReadTimeout),
           maxRetry = config.getInt(http.ahc.MaxRetry),
           requestTimeOut = config.getInt(http.ahc.RequestTimeout),
-          acceptAnyCertificate = {
-            val accept = config.getBoolean(http.ahc.AcceptAnyCertificate)
-            if (accept) {
+          disableHttpsAlgorithm = {
+            val disable = config.getBoolean(http.ahc.DisableHttpsAlgorithm)
+            if (disable) {
               System.setProperty("jdk.tls.allowUnsafeServerCertChange", "true")
               System.setProperty("sun.security.ssl.allowUnsafeRenegotiation", "true")
             }
-            accept
+            disable
           },
           httpClientCodecMaxChunkSize = config.getInt(http.ahc.HttpClientCodecMaxChunkSize),
           sslEnabledProtocols = config.getStringList(http.ahc.SslEnabledProtocols).asScala.toList,
@@ -339,7 +339,7 @@ case class AhcConfiguration(
   readTimeout:                         Int,
   maxRetry:                            Int,
   requestTimeOut:                      Int,
-  acceptAnyCertificate:                Boolean,
+  disableHttpsAlgorithm:               Boolean,
   httpClientCodecMaxChunkSize:         Int,
   sslEnabledProtocols:                 List[String],
   sslEnabledCipherSuites:              List[String],

gatling-http/src/main/scala/io/gatling/http/action/sync/AddCookieBuilder.scala

@@ -25,7 +25,7 @@ import io.gatling.http.cache.HttpCaches
 import io.gatling.http.cookie.CookieJar
 import io.gatling.http.cookie.CookieSupport.storeCookie
 
-import org.asynchttpclient.cookie.Cookie
+import io.netty.handler.codec.http.cookie.{ Cookie, DefaultCookie }
 import org.asynchttpclient.uri.Uri
 
 case class CookieDSL(name: Expression[String], value: Expression[String],
@@ -72,10 +72,7 @@ class AddCookieBuilder(name: Expression[String], value: Expression[String], doma
       value <- value(session)
       domain <- resolvedDomain(session)
       path <- resolvedPath(session)
-    } yield {
-      val cookie = new Cookie(name, value, false, domain, path, maxAge, false, false)
-      storeCookie(session, domain, path, cookie)
-    }
+    } yield storeCookie(session, domain, path, new DefaultCookie(name, value))
 
     new SessionHook(expression, genName("addCookie"), coreComponents.statsEngine, next) with ExitableAction
   }

gatling-http/src/main/scala/io/gatling/http/ahc/AhcFactory.scala

@@ -94,7 +94,7 @@ private[gatling] class DefaultAhcFactory(system: ActorSystem, coreComponents: Co
       .setEventLoopGroup(eventLoopGroup)
       .setNettyTimer(timer)
       .setResponseBodyPartFactory(ResponseBodyPartFactory.LAZY)
-      .setAcceptAnyCertificate(ahcConfig.acceptAnyCertificate)
+      .setDisableHttpsAlgorithm(ahcConfig.disableHttpsAlgorithm)
       .setEnabledProtocols(ahcConfig.sslEnabledProtocols match {
         case Nil => null
         case ps  => ps.toArray

gatling-http/src/main/scala/io/gatling/http/cache/Cookies.scala

@@ -20,11 +20,11 @@ import java.util.{ Collection => JCollection }
 import scala.collection.breakOut
 import scala.collection.JavaConverters._
 
-import org.asynchttpclient.cookie.Cookie
+import io.netty.handler.codec.http.cookie.Cookie
 
 class Cookies(cookies: JCollection[Cookie]) {
 
-  lazy val cookieNameValuePairs: Map[String, String] = cookies.asScala.map(cookie => cookie.getName -> cookie.getPath)(breakOut)
+  lazy val cookieNameValuePairs: Map[String, String] = cookies.asScala.map(cookie => cookie.name -> cookie.path)(breakOut)
 
   override def hashCode = cookieNameValuePairs.hashCode
 

gatling-http/src/main/scala/io/gatling/http/cache/ExpiresSupport.scala

@@ -20,7 +20,7 @@ import io.gatling.commons.util.ClockSingleton.unpreciseNowMillis
 import io.gatling.http.{ HeaderNames, HeaderValues }
 import io.gatling.http.response.Response
 
-import org.asynchttpclient.cookie.DateParser
+import io.netty.handler.codec.DateFormatter
 
 trait ExpiresSupport {
 
@@ -60,7 +60,7 @@ trait ExpiresSupport {
     // FIXME use offset instead of 2 substrings
     val trimmedTimeString = removeQuote(timestring.trim)
 
-    Option(DateParser.parse(trimmedTimeString)).map(_.getTime)
+    Option(DateFormatter.parseHttpDate(trimmedTimeString)).map(_.getTime)
   }
 
   def getResponseExpires(response: Response): Option[Long] = {

gatling-http/src/main/scala/io/gatling/http/cookie/CookieJar.scala

@@ -17,7 +17,7 @@ package io.gatling.http.cookie
 
 import io.gatling.commons.util.ClockSingleton._
 
-import org.asynchttpclient.cookie.Cookie
+import io.netty.handler.codec.http.cookie.Cookie
 import org.asynchttpclient.uri.Uri
 
 case class CookieKey(name: String, domain: String, path: String)
@@ -26,6 +26,7 @@ case class StoredCookie(cookie: Cookie, hostOnly: Boolean, persistent: Boolean,
 
 object CookieJar {
 
+  // FIXME to be replace with upcoming Netty constant in 4.1.9
   val UnspecifiedMaxAge = Long.MinValue
 
   def requestDomain(requestUri: Uri) = requestUri.getHost.toLowerCase
@@ -61,7 +62,7 @@ object CookieJar {
   }
 
   def hasExpired(c: Cookie): Boolean = {
-    val maxAge = c.getMaxAge
+    val maxAge = c.maxAge
     maxAge != CookieJar.UnspecifiedMaxAge && maxAge <= 0
   }
 
@@ -100,16 +101,16 @@ case class CookieJar(store: Map[CookieKey, StoredCookie]) {
     val newStore = cookies.foldLeft(store) {
       (updatedStore, cookie) =>
 
-        val (keyDomain, hostOnly) = cookieDomain(Option(cookie.getDomain), requestDomain)
+        val (keyDomain, hostOnly) = cookieDomain(Option(cookie.domain), requestDomain)
 
-        val keyPath = cookiePath(Option(cookie.getPath), requestPath)
+        val keyPath = cookiePath(Option(cookie.path), requestPath)
 
         if (hasExpired(cookie)) {
-          updatedStore - CookieKey(cookie.getName.toLowerCase, keyDomain, keyPath)
+          updatedStore - CookieKey(cookie.name.toLowerCase, keyDomain, keyPath)
 
         } else {
-          val persistent = cookie.getMaxAge != UnspecifiedMaxAge
-          updatedStore + (CookieKey(cookie.getName.toLowerCase, keyDomain, keyPath) -> StoredCookie(cookie, hostOnly, persistent, unpreciseNowMillis))
+          val persistent = cookie.maxAge != UnspecifiedMaxAge
+          updatedStore + (CookieKey(cookie.name.toLowerCase, keyDomain, keyPath) -> StoredCookie(cookie, hostOnly, persistent, unpreciseNowMillis))
         }
     }
 

gatling-http/src/main/scala/io/gatling/http/cookie/CookieSupport.scala

@@ -20,7 +20,7 @@ import io.gatling.core.session.{ Session, SessionPrivateAttributes }
 import io.gatling.core.session.Expression
 import io.gatling.http.util.HttpTypeHelper
 
-import org.asynchttpclient.cookie.Cookie
+import io.netty.handler.codec.http.cookie.Cookie
 import org.asynchttpclient.uri.Uri
 
 object CookieSupport {

gatling-http/src/main/scala/io/gatling/http/resolver/ExtendedDnsNameResolver.scala

@@ -21,7 +21,7 @@ import java.util.{ List => JList }
 import io.gatling.core.config.GatlingConfiguration
 
 import com.typesafe.scalalogging.StrictLogging
-import io.netty.bootstrap.ChannelFactory
+import io.netty.channel.ChannelFactory
 import io.netty.channel.EventLoop
 import io.netty.channel.socket.DatagramChannel
 import io.netty.channel.socket.nio.NioDatagramChannel
@@ -50,7 +50,7 @@ class ExtendedDnsNameResolver(eventLoop: EventLoop, configuration: GatlingConfig
       ExtendedDnsNameResolver.NioDatagramChannelFactory,
       DnsServerAddresses.defaultAddresses(),
       NoopDnsCache.INSTANCE,
-      NoopDnsCache.INSTANCE,
+      // FIXME Netty 4.1.9 NoopDnsCache.INSTANCE,
       configuration.http.dns.queryTimeout,
       NettyDnsConstants.DefaultResolveAddressTypes,
       true, // recursionDesired
@@ -60,10 +60,9 @@ class ExtendedDnsNameResolver(eventLoop: EventLoop, configuration: GatlingConfig
       true, // optResourceEnabled
       HostsFileEntriesResolver.DEFAULT,
       NettyDnsConstants.DefaultSearchDomain,
-      1, // ndots
-      true // decodeIdn
+      1 // ndots
+    // FIXME Netty 4.1.9 ,true // decodeIdn
     ) {
-
   override def doResolve(inetHost: String, additionals: Array[DnsRecord], promise: Promise[InetAddress], resolveCache: DnsCache): Unit =
     super.doResolve(inetHost, additionals, promise, resolveCache)
 

gatling-http/src/main/scala/io/gatling/http/response/Response.scala

@@ -20,16 +20,16 @@ import java.nio.charset.Charset
 import scala.collection.JavaConverters._
 
 import io.netty.handler.codec.http.HttpHeaders
-import org.asynchttpclient.cookie.{ Cookie, CookieDecoder }
 import org.asynchttpclient.netty.request.NettyRequest
 import org.asynchttpclient.{ HttpResponseStatus, Request => AHCRequest }
 import org.asynchttpclient.uri.Uri
-
 import io.gatling.core.stats.message.ResponseTimings
 import io.gatling.http.HeaderNames
 import io.gatling.http.protocol.HttpProtocol
 import io.gatling.http.util.HttpHelper
 
+import io.netty.handler.codec.http.cookie.{ ClientCookieDecoder, Cookie }
+
 abstract class Response {
 
   def request: AHCRequest
@@ -84,7 +84,7 @@ case class HttpResponse(
   def header(name: String): Option[String] = Option(headers.get(name))
   def headers(name: String): Seq[String] = headers.getAll(name).asScala
 
-  lazy val cookies = headers.getAll(HeaderNames.SetCookie).asScala.flatMap(cookie => Option(CookieDecoder.decode(cookie))).toList
+  lazy val cookies = headers.getAll(HeaderNames.SetCookie).asScala.flatMap(cookie => Option(ClientCookieDecoder.LAX.decode(cookie))).toList
 
   def checksum(algorithm: String) = checksums.get(algorithm)
   def hasResponseBody = bodyLength != 0

gatling-http/src/test/scala/io/gatling/http/HttpSpec.scala

@@ -100,7 +100,7 @@ abstract class HttpSpec extends AkkaSpec with BeforeAndAfter {
   def verifyRequestTo(path: String)(implicit server: HttpServer): Unit = verifyRequestTo(path, 1)
 
   def verifyRequestTo(path: String, count: Int, checks: (FullHttpRequest => Unit)*)(implicit server: HttpServer): Unit = {
-    val filteredRequests = server.requests.asScala.filter(_.getUri == path).toList
+    val filteredRequests = server.requests.asScala.filter(_.uri == path).toList
     val actualCount = filteredRequests.size
     if (actualCount != count) {
       throw new AssertionError(s"Expected to access $path $count times, but actually accessed it $actualCount times.")
@@ -126,7 +126,7 @@ abstract class HttpSpec extends AkkaSpec with BeforeAndAfter {
   // Extractor for nicer interaction with Scala
   class HttpRequest(val request: FullHttpRequest) {
     def isEmpty = request == null
-    def get: (HttpMethod, String) = (request.getMethod, request.getUri)
+    def get: (HttpMethod, String) = (request.method, request.uri)
   }
 
   object HttpRequest {

gatling-http/src/test/scala/io/gatling/http/cookie/CookieHandlingSpec.scala

@@ -18,7 +18,7 @@ package io.gatling.http.cookie
 import io.gatling.BaseSpec
 import io.gatling.core.session.Session
 
-import org.asynchttpclient.cookie.CookieDecoder.decode
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder.LAX.decode
 import org.asynchttpclient.uri.Uri
 
 class CookieHandlingSpec extends BaseSpec {
@@ -30,7 +30,7 @@ class CookieHandlingSpec extends BaseSpec {
     val originalDomain = "docs.foo.com"
     val originalCookieJar = new CookieJar(Map(CookieKey("ALPHA", originalDomain, "/") -> StoredCookie(originalCookie, hostOnly = true, persistent = true, 0L)))
     val originalSession = Session("scenarioName", 0, Map(CookieSupport.CookieJarAttributeName -> originalCookieJar))
-    CookieSupport.getStoredCookies(originalSession, "https://docs.foo.com/accounts").map(x => x.getValue) shouldBe List("VALUE1")
+    CookieSupport.getStoredCookies(originalSession, "https://docs.foo.com/accounts").map(x => x.value) shouldBe List("VALUE1")
   }
 
   it should "be called with an empty session" in {

gatling-http/src/test/scala/io/gatling/http/cookie/CookieJarSpec.scala

@@ -17,7 +17,7 @@ package io.gatling.http.cookie
 
 import io.gatling.BaseSpec
 
-import org.asynchttpclient.cookie.CookieDecoder.decode
+import io.netty.handler.codec.http.cookie.ClientCookieDecoder.LAX.decode
 import org.asynchttpclient.uri.Uri
 
 class CookieJarSpec extends BaseSpec {
@@ -88,7 +88,7 @@ class CookieJarSpec extends BaseSpec {
 
     val storedCookies = cookieStore.add(uri, List(decode("ALPHA=VALUE2; Domain=www.foo.com; path=/bar"))).get(uri)
     storedCookies should have size 1
-    storedCookies.head.getValue shouldBe "VALUE2"
+    storedCookies.head.value shouldBe "VALUE2"
   }
 
   it should "not replace cookies when they don't have the same name" in {
@@ -118,10 +118,10 @@ class CookieJarSpec extends BaseSpec {
 
     val storedCookies1 = cookieStore.get(uri1)
     storedCookies1 should have size 1
-    storedCookies1.head.getValue shouldBe "VALUE1"
+    storedCookies1.head.value shouldBe "VALUE1"
     val storedCookies2 = cookieStore.get(uri2)
     storedCookies2 should have size 1
-    storedCookies2.head.getValue shouldBe "VALUE2"
+    storedCookies2.head.value shouldBe "VALUE2"
   }
 
   it should "handle missing domain as request host" in {
@@ -178,7 +178,7 @@ class CookieJarSpec extends BaseSpec {
 
     val storedCookies = cookieStore.add(uri, List(decode("alpha=VALUE2; Domain=www.foo.com; path=/bar"))).get(uri)
     storedCookies should have size 1
-    storedCookies.head.getValue shouldBe "VALUE2"
+    storedCookies.head.value shouldBe "VALUE2"
   }
 
   it should "handle the cookie path in a case-sensitive manner (RFC 2965 sec. 3.3.3)" in {
@@ -212,7 +212,7 @@ class CookieJarSpec extends BaseSpec {
 
     val cookies = cookieStore.get(Uri.create("https://foo.org/"))
     cookies should have size 1
-    cookies.head.getValue shouldBe "VALUE3"
+    cookies.head.value shouldBe "VALUE3"
   }
 
   it should "should serve cookies based on the host and independently of the port" in {
@@ -225,7 +225,7 @@ class CookieJarSpec extends BaseSpec {
 
     val cookies = cookieStore2.get(Uri.create("http://foo.org/moodle/login"))
     cookies should have size 1
-    cookies.head.getValue shouldBe "VALUE2"
+    cookies.head.value shouldBe "VALUE2"
   }
 
   it should "properly deal with same name cookies" in {
@@ -240,13 +240,13 @@ class CookieJarSpec extends BaseSpec {
 
     val barCookies = cookieStore2.get(Uri.create("http://www.foo.com/foo/bar/"))
     barCookies should have size 2
-    barCookies(0).getValue shouldBe "VALUE1"
-    barCookies(1).getValue shouldBe "VALUE0"
+    barCookies(0).value shouldBe "VALUE1"
+    barCookies(1).value shouldBe "VALUE0"
 
     val bazCookies = cookieStore2.get(Uri.create("http://www.foo.com/foo/baz/"))
     bazCookies should have size 2
-    bazCookies(0).getValue shouldBe "VALUE2"
-    bazCookies(1).getValue shouldBe "VALUE0"
+    bazCookies(0).value shouldBe "VALUE2"
+    bazCookies(1).value shouldBe "VALUE0"
   }
 
   it should "properly deal with trailing slashes in paths" in {
@@ -257,6 +257,6 @@ class CookieJarSpec extends BaseSpec {
 
     val cookies = cookieStore.get(Uri.create("https://vagrant.moolb.com/app/consumer/"))
     cookies should have size 1
-    cookies(0).getValue shouldBe "211D17F016132BCBD31D9ABB31D90960"
+    cookies(0).value shouldBe "211D17F016132BCBD31D9ABB31D90960"
   }
 }

gatling-recorder/src/main/scala/io/gatling/recorder/http/ClientHandler.scala

@@ -15,12 +15,11 @@
  */
 package io.gatling.recorder.http
 
-import io.gatling.recorder.http.Netty._
 import io.gatling.recorder.http.flows.MitmMessage.{ ClientChannelException, ClientChannelInactive, ResponseReceived }
 
 import akka.actor.ActorRef
 import com.typesafe.scalalogging.StrictLogging
-import io.netty.channel.{ ChannelHandlerContext, ChannelInboundHandlerAdapter }
+import io.netty.channel.{ ChannelHandlerContext, ChannelId, ChannelInboundHandlerAdapter }
 import io.netty.handler.codec.http.FullHttpResponse
 
 class ClientHandler(mitmActor: ActorRef, serverChannelId: ChannelId, trafficLogger: TrafficLogger) extends ChannelInboundHandlerAdapter with StrictLogging {

gatling-recorder/src/main/scala/io/gatling/recorder/http/Netty.scala

@@ -23,8 +23,6 @@ import org.asynchttpclient.uri.Uri
 
 object Netty {
 
-  type ChannelId = Long
-
   implicit class PimpedChannelFuture(val cf: ChannelFuture) extends AnyVal {
 
     def addScalaListener(f: Try[Channel] => Unit): ChannelFuture =
@@ -43,8 +41,6 @@ object Netty {
 
   implicit class PimpedChannel(val channel: Channel) extends AnyVal {
 
-    def id: ChannelId = channel.hashCode
-
     def reply500AndClose(): Unit =
       channel
         .writeAndFlush(new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR))
@@ -54,8 +50,8 @@ object Netty {
   implicit class PimpedFullHttpRequest(val request: FullHttpRequest) extends AnyVal {
 
     def makeRelative: FullHttpRequest = {
-      val relativeUrl = Uri.create(request.getUri).toRelativeUrl
-      val relativeRequest = new DefaultFullHttpRequest(request.getProtocolVersion, request.getMethod, relativeUrl, request.content.retain())
+      val relativeUrl = Uri.create(request.uri).toRelativeUrl
+      val relativeRequest = new DefaultFullHttpRequest(request.protocolVersion, request.method, relativeUrl, request.content.retain())
       relativeRequest.headers.add(request.headers)
       relativeRequest
     }