Support for configuring if browser should run in headless mode or not

nlnwa · Oct 24, 2019 · 9c5bc76 · 9c5bc76
1 parent aa89148
commit 9c5bc76
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 5 deletions.
diff --git a/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/FrontierClient.java b/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/FrontierClient.java
@@ -45,6 +45,8 @@ public class FrontierClient implements AutoCloseable {
 
     private final Pool<ProxySession> pool;
 
+    private final boolean headlessBrowser;
+
     private static final String METRICS_NS = "veidemann";
     private static final String METRICS_SUBSYSTEM = "harvester";
 
@@ -86,18 +88,19 @@ public class FrontierClient implements AutoCloseable {
             .register();
 
     public FrontierClient(BrowserController controller, String host, int port, int maxOpenSessions,
-                          String browserWsEndpoint, String proxyHost, int firstProxyPort) {
+                          String browserWsEndpoint, String proxyHost, int firstProxyPort, boolean headlessBrowser) {
         this(controller, ManagedChannelBuilder.forAddress(host, port).usePlaintext(), maxOpenSessions,
-                browserWsEndpoint, proxyHost, firstProxyPort);
+                browserWsEndpoint, proxyHost, firstProxyPort, headlessBrowser);
     }
 
     /**
      * Construct client for accessing RouteGuide server using the existing channel.
      */
     public FrontierClient(BrowserController controller, ManagedChannelBuilder<?> channelBuilder, int maxOpenSessions,
-                          String browserWsEndpoint, String proxyHost, int firstProxyPort) {
+                          String browserWsEndpoint, String proxyHost, int firstProxyPort, boolean headlessBrowser) {
         LOG.info("Setting up Frontier client");
         this.controller = controller;
+        this.headlessBrowser = headlessBrowser;
         ClientTracingInterceptor tracingInterceptor = new ClientTracingInterceptor.Builder(GlobalTracer.get()).build();
         channel = channelBuilder.intercept(tracingInterceptor).build();
         asyncStub = FrontierGrpc.newStub(channel).withWaitForReady();
@@ -240,6 +243,7 @@ public ProxySession(int proxyId, String browserWSEndpoint, String proxyHost, int
                 query = query.add(proxyEntry);
             }
             query = query.add(new Entry("--ignore-certificate-errors", (String) null));
+            query = query.add(new Entry("headless", Boolean.toString(headlessBrowser)));
             browserWsEndpoint = UriConfigs.WHATWG.builder(ws).parsedQuery(query).build().toString();
             browserSessions.inc();
             LOG.info("Created session:  " + this);

diff --git a/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/Harvester.java b/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/Harvester.java
@@ -84,7 +84,7 @@ public Harvester start() {
 
              FrontierClient frontierClient = new FrontierClient(controller, SETTINGS.getFrontierHost(),
                      SETTINGS.getFrontierPort(), SETTINGS.getMaxOpenSessions(), SETTINGS.getBrowserWSEndpoint(),
-                     SETTINGS.getProxyHost(), SETTINGS.getProxyPort());
+                     SETTINGS.getProxyHost(), SETTINGS.getProxyPort(), SETTINGS.isHeadlessBrowser());
 
              RobotsServiceClient robotsServiceClient = new RobotsServiceClient(SETTINGS.getRobotsTxtEvaluatorHost(), SETTINGS.getRobotsTxtEvaluatorPort());
 

diff --git a/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/settings/Settings.java b/veidemann-harvester/src/main/java/no/nb/nna/veidemann/harvester/settings/Settings.java
@@ -55,6 +55,8 @@ public class Settings extends CommonSettings {
 
     private int robotsTxtEvaluatorPort;
 
+    private boolean headlessBrowser;
+
     public String getProxyHost() {
         return proxyHost;
     }
@@ -190,4 +192,12 @@ public int getRobotsTxtEvaluatorPort() {
     public void setRobotsTxtEvaluatorPort(int robotsTxtEvaluatorPort) {
         this.robotsTxtEvaluatorPort = robotsTxtEvaluatorPort;
     }
+
+    public boolean isHeadlessBrowser() {
+        return headlessBrowser;
+    }
+
+    public void setHeadlessBrowser(boolean headlessBrowser) {
+        this.headlessBrowser = headlessBrowser;
+    }
 }
diff --git a/veidemann-harvester/src/main/jib/app/resources/application.conf b/veidemann-harvester/src/main/jib/app/resources/application.conf
@@ -59,4 +59,7 @@ cachePort=3128
 cachePort=${?CACHE_PORT}
 
 maxOpenSessions=1
-maxOpenSessions=${?MAX_OPEN_SESSIONS}
+maxOpenSessions=${?MAX_OPEN_SESSIONS}
+
+headlessBrowser=true
+headlessBrowser=${?HEADLESS_BROWSER}