Get HTMLUnit tests working again

go.mod

@@ -8,5 +8,6 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
+	github.com/google/go-github/v27 v27.0.4 // indirect
 	google.golang.org/api v0.7.0
 )

go.sum

@@ -24,6 +24,10 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-github/v27 v27.0.4 h1:N/EEqsvJLgqTbepTiMBz+12KhwLovv6YvwpRezd+4Fg=
+github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -39,6 +43,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

remote.go

@@ -870,10 +870,7 @@ func (wd *remoteWD) SwitchFrame(frame interface{}) error {
 
 func (wd *remoteWD) ActiveElement() (WebElement, error) {
 	verb := "GET"
-	if wd.browser == "chrome" || (wd.browser == "firefox" && wd.browserVersion.Major < 47) {
-		// The W3C specification says that GET is the right verb, but Chrome
-		// implements only POST.
-		// https://github.com/seleniumhq/selenium/issues/2751
+	if wd.browser == "firefox" && wd.browserVersion.Major < 47 {
 		verb = "POST"
 	}
 	url := wd.requestURL("/session/%s/element/active", wd.id)

remote_test.go

@@ -36,6 +36,8 @@ var (
 	chromeDriverPath = flag.String("chrome_driver_path", "", "The path to the ChromeDriver binary. If empty or the file is not present, Chrome tests will not be run.")
 	chromeBinary     = flag.String("chrome_binary", "vendor/chrome-linux/chrome", "The name of the Chrome binary or the path to it. If name is not an exact path, the PATH will be searched.")
 
+	htmlUnitDriverPath = flag.String("htmlunit_driver_path", "vendor/htmlunit-driver.jar", "The path to the HTMLUnit Driver JAR.")
+
 	useDocker          = flag.Bool("docker", false, "If set, run the tests in a Docker container.")
 	runningUnderDocker = flag.Bool("running_under_docker", false, "This is set by the Docker test harness and should not be needed otherwise.")
 
@@ -256,9 +258,7 @@ func TestHTMLUnit(t *testing.T) {
 	c := config{
 		browser:         "htmlunit",
 		seleniumVersion: semver.MustParse("3.0.0"),
-	}
-	if *startFrameBuffer {
-		c.serviceOptions = append(c.serviceOptions, StartFrameBuffer())
+		serviceOptions:  []ServiceOption{HTMLUnit(*htmlUnitDriverPath)},
 	}
 
 	port, err := pickUnusedPort()
@@ -464,6 +464,9 @@ func newTestCapabilities(t *testing.T, c config) Capabilities {
 				// This flag is needed to test against Chrome binaries that are not the
 				// default installation. The sandbox requires a setuid binary.
 				"--no-sandbox",
+				// Allow Chrome to use the specified proxy for localhost, which is
+				// needed for the Proxy test. https://crbug.com/899126
+				"--proxy-bypass-list=<-loopback>",
 			},
 		}
 		caps.AddChrome(chrCaps)
@@ -616,7 +619,7 @@ func testNewSession(t *testing.T, c config) {
 		t.Fatalf("Got session id mismatch %s != %s", sid, wd.SessionID())
 	}
 
-	if wd.browserVersion.Major == 0 {
+	if c.browser != "htmlunit" && wd.browserVersion.Major == 0 {
 		t.Fatalf("wd.browserVersion.Major = %d, expected > 0", wd.browserVersion.Major)
 	}
 }
@@ -685,6 +688,8 @@ func testExtendedErrorMessage(t *testing.T, c config) {
 	}
 }
 
+// TODO(ekg): does this method work anymore in any browser? It is not part of
+// the W3C standard.
 func testCapabilities(t *testing.T, c config) {
 	if c.browser == "firefox" {
 		t.Skip("This method is not supported by Geckodriver.")
@@ -1595,7 +1600,7 @@ func testActiveElement(t *testing.T, c config) {
 		t.Fatalf("wd.ActiveElement().GetAttribute() returned error: %v", err)
 	}
 	if name != "q" {
-		t.Fatalf("wd.ActiveElement().GetAttribute() returned element wht name = %q, expected name = 'q'", name)
+		t.Fatalf("wd.ActiveElement().GetAttribute() returned element with name = %q, expected name = 'q'", name)
 	}
 }
 

service.go

@@ -114,21 +114,34 @@ func GeckoDriver(path string) ServiceOption {
 	}
 }
 
+// ChromeDriver sets the path for Chromedriver for the Selenium Server.  This
+// ServiceOption is only useful when calling NewSeleniumService.
 func ChromeDriver(path string) ServiceOption {
 	return func(s *Service) error {
 		s.chromeDriverPath = path
 		return nil
 	}
 }
 
-// JavaPath specifies the path to the JRE for a Selenium service.
+// JavaPath specifies the path to the JRE.
 func JavaPath(path string) ServiceOption {
 	return func(s *Service) error {
 		s.javaPath = path
 		return nil
 	}
 }
 
+// HTMLUnit specifies the path to the JAR for the HTMLUnit driver (compiled
+// with its dependencies).
+//
+// https://github.com/SeleniumHQ/htmlunit-driver/releases
+func HTMLUnit(path string) ServiceOption {
+	return func(s *Service) error {
+		s.htmlUnitPath = path
+		return nil
+	}
+}
+
 // Service controls a locally-running Selenium subprocess.
 type Service struct {
 	port            int
@@ -141,6 +154,7 @@ type Service struct {
 
 	geckoDriverPath, javaPath string
 	chromeDriverPath          string
+	htmlUnitPath              string
 
 	output io.Writer
 }
@@ -152,20 +166,28 @@ func (s Service) FrameBuffer() *FrameBuffer {
 
 // NewSeleniumService starts a Selenium instance in the background.
 func NewSeleniumService(jarPath string, port int, opts ...ServiceOption) (*Service, error) {
-	cmd := exec.Command("java", "-jar", jarPath, "-port", strconv.Itoa(port), "-debug")
-	s, err := newService(cmd, "/wd/hub", port, opts...)
+	s, err := newService(exec.Command("java"), "/wd/hub", port, opts...)
 	if err != nil {
 		return nil, err
 	}
 	if s.javaPath != "" {
 		s.cmd.Path = s.javaPath
 	}
 	if s.geckoDriverPath != "" {
-		s.cmd.Args = append([]string{"java", "-Dwebdriver.gecko.driver=" + s.geckoDriverPath}, cmd.Args[1:]...)
+		s.cmd.Args = append([]string{"java", "-Dwebdriver.gecko.driver=" + s.geckoDriverPath}, s.cmd.Args[1:]...)
 	}
 	if s.chromeDriverPath != "" {
-		s.cmd.Args = append([]string{"java", "-Dwebdriver.chrome.driver=" + s.chromeDriverPath}, cmd.Args[1:]...)
+		s.cmd.Args = append([]string{"java", "-Dwebdriver.chrome.driver=" + s.chromeDriverPath}, s.cmd.Args[1:]...)
 	}
+
+	var classpath []string
+	if s.htmlUnitPath != "" {
+		classpath = append(classpath, s.htmlUnitPath)
+	}
+	classpath = append(classpath, jarPath)
+	s.cmd.Args = append(s.cmd.Args, "-cp", strings.Join(classpath, ":"))
+	s.cmd.Args = append(s.cmd.Args, "org.openqa.grid.selenium.GridLauncherV3", "-port", strconv.Itoa(port), "-debug")
+
 	if err := s.start(port); err != nil {
 		return nil, err
 	}

vendor/init.go

@@ -19,11 +19,13 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"strings"
 	"sync"
 
 	"cloud.google.com/go/storage"
 	"github.com/golang/glog"
+	"github.com/google/go-github/v27/github"
 	"google.golang.org/api/option"
 )
 
@@ -76,6 +78,38 @@ var files = []file{
 	},
 }
 
+// addLatestGithubRelease adds a file to the list of files to download from the
+// latest release of the specified Github repository that matches the asset
+// name. The file will be downloaded to localFileName.
+func addLatestGithubRelease(ctx context.Context, owner, repo, assetName, localFileName string) error {
+	client := github.NewClient(nil)
+
+	rel, _, err := client.Repositories.GetLatestRelease(ctx, owner, repo)
+	if err != nil {
+		return err
+	}
+	assetNameRE, err := regexp.Compile(assetName)
+	if err != nil {
+		return fmt.Errorf("invalid asset name regular expression %q: %s", assetName, err)
+	}
+	for _, a := range rel.Assets {
+		if !assetNameRE.MatchString(a.GetName()) {
+			continue
+		}
+		u := a.GetBrowserDownloadURL()
+		if u == "" {
+			return fmt.Errorf("%s does not have a download URL", a.Name)
+		}
+		files = append(files, file{
+			name: localFileName,
+			url:  u,
+		})
+		return nil
+	}
+
+	return fmt.Errorf("Release for %s not found at http://github.com/%s/%s/releases", assetName, owner, repo)
+}
+
 // addChrome adds the appropriate chromium files to the list.
 //
 // If `latestChromeBuild` is empty, then the latest build will be used.
@@ -167,10 +201,15 @@ func main() {
 		}
 
 		if err := addChrome(ctx, chromeBuild); err != nil {
-			glog.Errorf("unable to Download Google Chrome browser: %v", err)
+			glog.Errorf("Unable to download Google Chrome browser: %v", err)
 		}
 		addFirefox(firefoxVersion)
 	}
+
+	if err := addLatestGithubRelease(ctx, "SeleniumHQ", "htmlunit-driver", "htmlunit-driver-.*-jar-with-dependencies.jar", "htmlunit-driver.jar"); err != nil {
+		glog.Errorf("Unable to download HTMLUnit Driver: %s", err)
+	}
+
 	var wg sync.WaitGroup
 	for _, file := range files {
 		wg.Add(1)