Add support for Blacklisting some domains for HTTPInputSource (#10535)

fix inspections

refactor class name

change name

 add allowList as well

distinguish between empty and null list

Fix CI

core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java

@@ -19,6 +19,7 @@
 
 package org.apache.druid.data.input.impl;
 
+import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
@@ -28,6 +29,7 @@
 import org.apache.druid.data.input.InputSourceReader;
 import org.apache.druid.data.input.InputSplit;
 import org.apache.druid.data.input.SplitHintSpec;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.metadata.PasswordProvider;
 
 import javax.annotation.Nullable;
@@ -46,17 +48,25 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
   @Nullable
   private final PasswordProvider httpAuthenticationPasswordProvider;
 
+  private final HttpInputSourceConfig config;
+
   @JsonCreator
   public HttpInputSource(
       @JsonProperty("uris") List<URI> uris,
       @JsonProperty("httpAuthenticationUsername") @Nullable String httpAuthenticationUsername,
-      @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider
+      @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider,
+      @JacksonInject HttpInputSourceConfig config
   )
   {
     Preconditions.checkArgument(uris != null && !uris.isEmpty(), "Empty URIs");
+    uris.forEach(uri -> Preconditions.checkArgument(
+        config.isURIAllowed(uri),
+        StringUtils.format("Access to [%s] DENIED!", uri)
+    ));
     this.uris = uris;
     this.httpAuthenticationUsername = httpAuthenticationUsername;
     this.httpAuthenticationPasswordProvider = httpAuthenticationPasswordProvider;
+    this.config = config;
   }
 
   @JsonProperty
@@ -97,7 +107,8 @@ public SplittableInputSource<URI> withSplit(InputSplit<URI> split)
     return new HttpInputSource(
         Collections.singletonList(split.get()),
         httpAuthenticationUsername,
-        httpAuthenticationPasswordProvider
+        httpAuthenticationPasswordProvider,
+        config
     );
   }
 

core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Objects;
+
+public class HttpInputSourceConfig
+{
+  @JsonProperty
+  private final List<String> allowListDomains;
+  @JsonProperty
+  private final List<String> denyListDomains;
+
+  @JsonCreator
+  public HttpInputSourceConfig(
+      @JsonProperty("allowListDomains") List<String> allowListDomains,
+      @JsonProperty("denyListDomains") List<String> denyListDomains
+  )
+  {
+    this.allowListDomains = allowListDomains;
+    this.denyListDomains = denyListDomains;
+    Preconditions.checkArgument(
+        this.denyListDomains == null || this.allowListDomains == null,
+        "Can only use one of allowList or blackList"
+    );
+  }
+
+  public List<String> getAllowListDomains()
+  {
+    return allowListDomains;
+  }
+
+  public List<String> getDenyListDomains()
+  {
+    return denyListDomains;
+  }
+
+  private static boolean matchesAny(List<String> domains, URI uri)
+  {
+    for (String domain : domains) {
+      if (uri.getHost().endsWith(domain)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public boolean isURIAllowed(URI uri)
+  {
+    if (allowListDomains != null) {
+      return matchesAny(allowListDomains, uri);
+    }
+    if (denyListDomains != null) {
+      return !matchesAny(denyListDomains, uri);
+    }
+    // No blacklist/allowList configured, all URLs are allowed
+    return true;
+  }
+
+  @Override
+  public String toString()
+  {
+    return "HttpInputSourceConfig{" +
+           "allowListDomains=" + allowListDomains +
+           ", denyListDomains=" + denyListDomains +
+           '}';
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    HttpInputSourceConfig config = (HttpInputSourceConfig) o;
+    return Objects.equals(allowListDomains, config.allowListDomains) &&
+           Objects.equals(denyListDomains, config.denyListDomains);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hash(allowListDomains, denyListDomains);
+  }
+}
+

core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java

@@ -19,6 +19,7 @@
 
 package org.apache.druid.data.input.impl;
 
+import com.fasterxml.jackson.databind.InjectableValues;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import org.apache.druid.data.input.InputSource;
@@ -28,20 +29,91 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.Collections;
 
 public class HttpInputSourceTest
 {
   @Test
   public void testSerde() throws IOException
   {
+    HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
+        null,
+        null
+    );
     final ObjectMapper mapper = new ObjectMapper();
+    mapper.setInjectableValues(new InjectableValues.Std().addValue(
+        HttpInputSourceConfig.class,
+        httpInputSourceConfig
+    ));
     final HttpInputSource source = new HttpInputSource(
         ImmutableList.of(URI.create("http://test.com/http-test")),
         "myName",
-        new DefaultPasswordProvider("myPassword")
+        new DefaultPasswordProvider("myPassword"),
+        httpInputSourceConfig
     );
     final byte[] json = mapper.writeValueAsBytes(source);
     final HttpInputSource fromJson = (HttpInputSource) mapper.readValue(json, InputSource.class);
     Assert.assertEquals(source, fromJson);
   }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testDenyListDomainThrowsException()
+  {
+    new HttpInputSource(
+        ImmutableList.of(URI.create("http://deny.com/http-test")),
+        "myName",
+        new DefaultPasswordProvider("myPassword"),
+        new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
+    );
+  }
+
+  @Test
+  public void testDenyListDomainNoMatch()
+  {
+    new HttpInputSource(
+        ImmutableList.of(URI.create("http://allow.com/http-test")),
+        "myName",
+        new DefaultPasswordProvider("myPassword"),
+        new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
+    );
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testAllowListDomainThrowsException()
+  {
+    new HttpInputSource(
+        ImmutableList.of(URI.create("http://deny.com/http-test")),
+        "myName",
+        new DefaultPasswordProvider("myPassword"),
+        new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
+    );
+  }
+
+  @Test
+  public void testAllowListDomainMatch()
+  {
+    new HttpInputSource(
+        ImmutableList.of(URI.create("http://allow.com/http-test")),
+        "myName",
+        new DefaultPasswordProvider("myPassword"),
+        new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
+    );
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testEmptyAllowListDomainMatch()
+  {
+    new HttpInputSource(
+        ImmutableList.of(URI.create("http://allow.com/http-test")),
+        "myName",
+        new DefaultPasswordProvider("myPassword"),
+        new HttpInputSourceConfig(Collections.emptyList(), null)
+    );
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testCannotSetBothAllowAndDenyList()
+  {
+    new HttpInputSourceConfig(Collections.singletonList("allow.com"), Collections.singletonList("deny.com"));
+  }
 }

docs/configuration/index.md

@@ -1353,6 +1353,15 @@ The amount of direct memory needed by Druid is at least
 ensure at least this amount of direct memory is available by providing `-XX:MaxDirectMemorySize=<VALUE>` at the command
 line.
 
+#### Indexer Security Configuration
+You can optionally configure following additional configs to restrict druid ingestion
+
+|Property|Possible Values|Description|Default|
+|--------|---------------|-----------|-------|
+|`druid.ingestion.http.allowListDomains`|List of domains|Allowed domains from which ingestion will be allowed. Only one of allowList or denyList can be set.|empty list|
+|`druid.ingestion.http.denyListDomains`|List of domains|Blacklisted domains from which ingestion will NOT be allowed. Only one of allowList or denyList can be set. |empty list|
+
+
 #### Query Configurations
 
 See [general query configuration](#general-query-configuration).

server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java

@@ -24,6 +24,8 @@
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Binder;
+import org.apache.druid.data.input.impl.HttpInputSourceConfig;
+import org.apache.druid.guice.JsonConfigProvider;
 import org.apache.druid.initialization.DruidModule;
 
 import java.util.List;
@@ -47,5 +49,6 @@ public List<? extends Module> getJacksonModules()
   @Override
   public void configure(Binder binder)
   {
+    JsonConfigProvider.bind(binder, "druid.ingestion.http", HttpInputSourceConfig.class);
   }
 }

server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java

@@ -25,12 +25,27 @@
 import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
 import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver;
 import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.ProvisionException;
+import org.apache.druid.data.input.impl.HttpInputSourceConfig;
+import org.apache.druid.guice.DruidGuiceExtensions;
+import org.apache.druid.guice.JsonConfigurator;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.guice.LifecycleModule;
+import org.apache.druid.guice.ServerModule;
+import org.apache.druid.jackson.JacksonModule;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import javax.validation.Validation;
+import javax.validation.Validator;
+import java.util.Collections;
 import java.util.List;
+import java.util.Properties;
 import java.util.stream.Collectors;
 
 public class InputSourceModuleTest
@@ -41,7 +56,8 @@ public class InputSourceModuleTest
   @Before
   public void setUp()
   {
-    for (Module jacksonModule : new InputSourceModule().getJacksonModules()) {
+    InputSourceModule inputSourceModule = new InputSourceModule();
+    for (Module jacksonModule : inputSourceModule.getJacksonModules()) {
       mapper.registerModule(jacksonModule);
     }
   }
@@ -59,4 +75,62 @@ public void testSubTypeRegistration()
     Assert.assertNotNull(subtypes);
     Assert.assertEquals(SQL_NAMED_TYPE, Iterables.getOnlyElement(subtypes));
   }
+
+  @Test
+  public void testHttpInputSourceAllowConfig()
+  {
+    Properties props = new Properties();
+    props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
+    Injector injector = makeInjectorWithProperties(props);
+    HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
+    Assert.assertEquals(new HttpInputSourceConfig(Collections.singletonList("allow.com"), null), instance);
+  }
+
+  @Test
+  public void testHttpInputSourceDenyConfig()
+  {
+    Properties props = new Properties();
+    props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
+    Injector injector = makeInjectorWithProperties(props);
+    HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
+    Assert.assertEquals(new HttpInputSourceConfig(null, Collections.singletonList("deny.com")), instance);
+  }
+
+  @Test(expected = ProvisionException.class)
+  public void testHttpInputSourceBothAllowDenyConfig()
+  {
+    Properties props = new Properties();
+    props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
+    props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
+    Injector injector = makeInjectorWithProperties(props);
+    injector.getInstance(HttpInputSourceConfig.class);
+  }
+
+  @Test
+  public void testHttpInputSourceDefaultConfig()
+  {
+    Properties props = new Properties();
+    Injector injector = makeInjectorWithProperties(props);
+    HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
+    Assert.assertEquals(new HttpInputSourceConfig(null, null), instance);
+    Assert.assertNull(instance.getAllowListDomains());
+    Assert.assertNull(instance.getDenyListDomains());
+  }
+
+  private Injector makeInjectorWithProperties(final Properties props)
+  {
+    return Guice.createInjector(
+        ImmutableList.of(
+            new DruidGuiceExtensions(),
+            new LifecycleModule(),
+            new ServerModule(),
+            binder -> {
+              binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
+              binder.bind(JsonConfigurator.class).in(LazySingleton.class);
+              binder.bind(Properties.class).toInstance(props);
+            },
+            new JacksonModule(),
+            new InputSourceModule()
+        ));
+  }
 }