Skip to content

upgrade basepom, apply prettier formatting updates #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jaredstehler merged 4 commits into from
Mar 12, 2024
Merged

upgrade basepom, apply prettier formatting updates #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
186e846
upgrade basepom, apply prettier formatting updates
jaredstehler Mar 10, 2024
126e5ec
use release basepom
jaredstehler Mar 10, 2024
7150ffb
moving spotbugs configs to basepom
jaredstehler Mar 12, 2024
45672f3
Merge branch 'master' into js-update-basepom
jaredstehler Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file removed .build-jdk11
View file
Open in desktop
Empty file.
88 changes: 35 additions & 53 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,107 +5,89 @@
<parent>
<groupId>com.hubspot</groupId>
<artifactId>basepom</artifactId>
<version>25.3</version>
<version>59.3</version>
</parent>

<groupId>com.hubspot.slimfast</groupId>
<artifactId>slimfast-parent</artifactId>
<version>0.23-SNAPSHOT</version>
<packaging>pom</packaging>

<prerequisites>
<maven>3</maven>
</prerequisites>

<modules>
<module>slimfast-plugin</module>
<module>slimfast-hadoop</module>
</modules>

<properties>
<aws.sdk.version>1.11.461</aws.sdk.version>
<dep.httpclient.version>4.5.5</dep.httpclient.version>
<dep.httpcore.version>4.4.9</dep.httpcore.version>
<dep.maven.version>3.6.3</dep.maven.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.hubspot</groupId>
<artifactId>failsafe-s3-decorator</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
<exclusions>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-archiver</artifactId>
<version>3.0.0</version>
<version>3.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
<version>3.3.3</version>
<version>${dep.maven.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>3.3.3</version>
<version>${dep.maven.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>3.3.3</version>
<version>${dep.maven.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.3.3</version>
<version>${dep.maven.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
<version>3.4</version>
<version>3.11.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-interpolation</artifactId>
<version>1.22</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>com.hubspot</groupId>
<artifactId>failsafe-s3-decorator</artifactId>
<version>0.7</version>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
<exclusions>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</exclusion>
</exclusions>
<version>1.25</version>
</dependency>
</dependencies>
</dependencyManagement>

<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-plugin-plugin</artifactId>
<version>3.4</version>
</plugin>
</plugins>
</pluginManagement>
</build>

<developers>
<developer>
<name>Jonathan Haber</name>
Expand Down
48 changes: 32 additions & 16 deletions slimfast-hadoop/src/main/java/com/hubspot/slimfast/hadoop/HadoopHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
package com.hubspot.slimfast.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -15,20 +8,33 @@
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.jar.Manifest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class HadoopHelper {

private static final Logger LOG = LoggerFactory.getLogger(HadoopHelper.class);

public static void writeJarsToHdfsAndAddToClasspath(SlimfastHadoopConfiguration slimfastConfiguration) {
public static void writeJarsToHdfsAndAddToClasspath(
SlimfastHadoopConfiguration slimfastConfiguration
) {
try {
FileSystem hdfs = FileSystem.get(slimfastConfiguration.getConfiguration());

for (String jar : findClasspathJars(slimfastConfiguration.getJarDirectory())) {
Path destination = new Path(slimfastConfiguration.getHdfsArtifactRoot().resolve(jar).toString());
Path destination = new Path(
slimfastConfiguration.getHdfsArtifactRoot().resolve(jar).toString()
);
if (exists(hdfs, destination)) {
LOG.info("Path already exists {}", destination);
} else {
Path source = new Path(slimfastConfiguration.getJarDirectory().resolve(jar).toString());
Path source = new Path(
slimfastConfiguration.getJarDirectory().resolve(jar).toString()
);
hdfs.copyFromLocalFile(source, destination);
LOG.info("Successfully uploaded path {}", destination);
}
Expand All @@ -40,15 +46,20 @@ public static void writeJarsToHdfsAndAddToClasspath(SlimfastHadoopConfiguration
}
}

private static void addJarToJobConfiguration(Path jarPath, Configuration configuration) throws IOException {
private static void addJarToJobConfiguration(Path jarPath, Configuration configuration)
throws IOException {
String jar = jarPath.toString();
String existingClasspath = configuration.get("mapreduce.job.classpath.files");
String updatedClasspath = existingClasspath == null ? jar : existingClasspath + "," + jar;
String updatedClasspath = existingClasspath == null
? jar
: existingClasspath + "," + jar;
configuration.set("mapreduce.job.classpath.files", updatedClasspath);

jar = FileSystem.get(configuration).makeQualified(jarPath).toUri().toString();
String existingCacheFiles = configuration.get("mapreduce.job.cache.files");
String updatedCacheFiles = existingCacheFiles == null ? jar : existingCacheFiles + "," + jar;
String updatedCacheFiles = existingCacheFiles == null
? jar
: existingCacheFiles + "," + jar;
configuration.set("mapreduce.job.cache.files", updatedCacheFiles);
}

Expand All @@ -61,9 +72,12 @@ private static boolean exists(FileSystem fileSystem, Path path) throws IOExcepti
}
}

private static Set<String> findClasspathJars(java.nio.file.Path jarDirectory) throws IOException {
private static Set<String> findClasspathJars(java.nio.file.Path jarDirectory)
throws IOException {
Set<String> classpathJars = new LinkedHashSet<>();
for (URL url : Collections.list(getClassLoader().getResources("META-INF/MANIFEST.MF"))) {
for (URL url : Collections.list(
getClassLoader().getResources("META-INF/MANIFEST.MF")
)) {
try (InputStream manifestStream = url.openStream()) {
Manifest manifest = new Manifest(manifestStream);
String classPath = manifest.getMainAttributes().getValue("Class-Path");
Expand All @@ -82,6 +96,8 @@ private static Set<String> findClasspathJars(java.nio.file.Path jarDirectory) th

private static ClassLoader getClassLoader() {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
return contextClassLoader == null ? HadoopHelper.class.getClassLoader() : contextClassLoader;
return contextClassLoader == null
? HadoopHelper.class.getClassLoader()
: contextClassLoader;
}
}
28 changes: 22 additions & 6 deletions slimfast-hadoop/src/main/java/com/hubspot/slimfast/hadoop/SlimfastHadoopConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;

import org.apache.hadoop.conf.Configuration;

public class SlimfastHadoopConfiguration {

private final Path jarDirectory;
private final Path hdfsArtifactRoot;
private final Configuration configuration;

private SlimfastHadoopConfiguration(Path jarDirectory, Path hdfsArtifactRoot, Configuration configuration) {
private SlimfastHadoopConfiguration(
Path jarDirectory,
Path hdfsArtifactRoot,
Configuration configuration
) {
this.jarDirectory = jarDirectory;
this.hdfsArtifactRoot = hdfsArtifactRoot;
this.configuration = configuration;
Expand All @@ -34,22 +38,30 @@ public Configuration getConfiguration() {
}

public static class Builder {

private Path jarDirectory;
private Path hdfsArtifactRoot = Paths.get("jars");
private Configuration configuration;

public Builder setJarByClass(Class<?> jarClass) {
URL url = jarClass.getResource("/" + jarClass.getName().replace('.', '/') + ".class");
URL url = jarClass.getResource(
"/" + jarClass.getName().replace('.', '/') + ".class"
);
if (url == null) {
throw new IllegalStateException("Could not find resource " + jarClass);
}

String qualifiedPath = url.toString();
if (!qualifiedPath.startsWith("jar:file:")) {
throw new IllegalStateException("Class doesn't appear to be in a JAR, are you running from a JAR?");
throw new IllegalStateException(
"Class doesn't appear to be in a JAR, are you running from a JAR?"
);
}

String jarPath = qualifiedPath.substring("jar:file:".length(), qualifiedPath.indexOf('!'));
String jarPath = qualifiedPath.substring(
"jar:file:".length(),
qualifiedPath.indexOf('!')
);
return setJarDirectory(Paths.get(jarPath).getParent());
}

Expand Down Expand Up @@ -77,7 +89,11 @@ public SlimfastHadoopConfiguration build() {
throw new IllegalStateException("configuration must be set");
}

return new SlimfastHadoopConfiguration(jarDirectory, hdfsArtifactRoot, configuration);
return new SlimfastHadoopConfiguration(
jarDirectory,
hdfsArtifactRoot,
configuration
);
}
}
}
Loading