Skip to content
Permalink
Browse files
FLUME-3298 Make hadoop-common optional in hadoop-credential-store-con…
…fig-filter

hadoop-common should be optional as in the hdfs-sink

This closes #248

Reviewers: Endre Major, Peter Turcsanyi

(Ferenc Szabo via Ferenc Szabo)
  • Loading branch information
szaboferee committed Nov 27, 2018
1 parent 85e0b4e commit 4ae19e8f9701399540501256ec0cce29254083fd
Showing 5 changed files with 41 additions and 4 deletions.
@@ -33,6 +33,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
@@ -4782,6 +4782,9 @@ The ``generateUniqId.sh`` will return ``1234`` with an exit code ``0``.
Hadoop Credential Store Config Filter
-------------------------------------

A hadoop-common library needed on the classpath for this feature (2.6+ version).
If hadoop is installed the agent adds it to the classpath automatically

=============================================== ========== ==============================================
Property Name Default Description
=============================================== ========== ==============================================
@@ -74,6 +74,22 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<outputFile>${project.build.directory}/test-classes/classpath.txt</outputFile>
</configuration>
<executions>
<execution>
<id>generate-classpath</id>
<phase>generate-test-resources</phase>
<goals>
<goal>build-classpath</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -33,9 +33,14 @@

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Scanner;
import java.util.Set;
@@ -54,6 +59,7 @@ public class TestConfig {

private Properties agentProps;
private Map<String, String> agentEnv;
private Map<String, String> agentOptions;
private File sinkOutputDir1;
private File sinkOutputDir2;
private File sinkOutputDir3;
@@ -73,6 +79,8 @@ public void setup() throws Exception {

agentProps = new Properties();
agentEnv = new HashMap<>();
agentOptions = new HashMap<>();
agentOptions.put("-C", getAdditionalClassPath());

// Create the rest of the properties file
agentProps.put("agent.sources.seq-01.type", "seq");
@@ -135,6 +143,12 @@ public void setup() throws Exception {
agentProps.put("agent.configfilters", "filter-01 filter-02 filter-03");
}

private String getAdditionalClassPath() throws Exception {
URL resource = this.getClass().getClassLoader().getResource("classpath.txt");
Path path = Paths.get(Objects.requireNonNull(resource).getPath());
return Files.readAllLines(path).stream().findFirst().orElse("");
}

@After
public void teardown() throws Exception {
StagedInstall.getInstance().stopAgent();
@@ -164,7 +178,7 @@ private void validateSeenEvents(File outDir, int outFiles, int events)
public void testConfigReplacement() throws Exception {
LOGGER.debug("testConfigReplacement() started.");

StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv);
StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv, agentOptions);

TimeUnit.SECONDS.sleep(10); // Wait for sources and sink to process files

@@ -183,7 +197,7 @@ public void testConfigReload() throws Exception {

agentProps.put("agent.channels.mem-01.transactionCapacity", "10");
agentProps.put("agent.sinks.roll-01.sink.batchSize", "20");
StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv);
StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv, agentOptions);

TimeUnit.SECONDS.sleep(10); // Wait for sources and sink to process files

@@ -123,11 +123,12 @@ public synchronized int startAgent(String name, String configResource)
}

public synchronized void startAgent(String name, Properties properties) throws Exception {
startAgent(name, properties, new HashMap<>());
startAgent(name, properties, new HashMap<>(), new HashMap<>());
}

public synchronized void startAgent(
String name, Properties properties, Map<String, String> environmentVariables)
String name, Properties properties, Map<String, String> environmentVariables,
Map<String, String> commandOptions)
throws Exception {
Preconditions.checkArgument(!name.isEmpty(), "agent name must not be empty");
Preconditions.checkNotNull(properties, "properties object must not be null");
@@ -159,6 +160,8 @@ public synchronized void startAgent(
+ ENV_FLUME_ROOT_LOGGER_VALUE);
builder.add("-D" + ENV_FLUME_LOG_FILE + "=" + logFileName);

commandOptions.forEach((key, value) -> builder.add(key, value));

List<String> cmdArgs = builder.build();

LOGGER.info("Using command: " + Joiner.on(" ").join(cmdArgs));

0 comments on commit 4ae19e8

Please sign in to comment.