Skip to content
Permalink
Browse files
Analyze dependencies and fix spark/pom.xml
* Add dependency analysis to POMs
* Update the spark/pom.xml (this fixes #93)
* Fix compilation issue with CopyPlus5K, which had missing imports
  • Loading branch information
ctubbsii committed Apr 1, 2022
1 parent 7970495 commit 88ee1ecf8695e57e8ca4d720d7917af6e279f456
Showing 3 changed files with 88 additions and 52 deletions.
39 pom.xml
@@ -77,31 +77,34 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core4</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
@@ -150,11 +153,31 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.3.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>analyze</id>
<goals>
<goal>analyze-only</goal>
</goals>
<configuration>
<failOnWarning>true</failOnWarning>
<ignoredUsedUndeclaredDependencies />
<ignoredUnusedDeclaredDependencies>
<runtime>org.apache.logging.log4j:*</runtime>
</ignoredUnusedDeclaredDependencies>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
@@ -20,108 +20,121 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>23</version>
<version>24</version>
<relativePath />
</parent>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-spark</artifactId>
<version>2.0.0-SNAPSHOT</version>
<version>2.1.0-SNAPSHOT</version>
<name>Apache Accumulo Spark Example</name>
<description>Example Spark Application for Apache Accumulo</description>
<properties>
<accumulo.version>2.0.0</accumulo.version>
<accumulo.version>2.1.0-SNAPSHOT</accumulo.version>
<eclipseFormatterStyle>contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
<hadoop.version>3.2.1</hadoop.version>
<maven.compiler.release>8</maven.compiler.release>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<zookeeper.version>3.4.14</zookeeper.version>
<hadoop.version>3.3.0</hadoop.version>
<maven.compiler.release>11</maven.compiler.release>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<minimalJavaBuildVersion>11</minimalJavaBuildVersion>
<minimalMavenBuildVersion>3.5.0</minimalMavenBuildVersion>
<!-- timestamp for reproducible outputs, updated on release by the release plugin -->
<project.build.outputTimestamp>2020-12-17T22:06:50Z</project.build.outputTimestamp>
<zookeeper.version>3.7.0</zookeeper.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-project</artifactId>
<version>${accumulo.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
<version>${accumulo.version}</version>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-hadoop-mapreduce</artifactId>
<version>${accumulo.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.11.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<optimize>true</optimize>
</configuration>
</plugin>
<plugin>
<groupId>com.github.ekryd.sortpom</groupId>
<artifactId>sortpom-maven-plugin</artifactId>
<version>2.10.0</version>
<version>3.0.0</version>
<configuration>
<predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
<createBackupFile>false</createBackupFile>
<lineSeparator>\n</lineSeparator>
<expandEmptyElements>false</expandEmptyElements>
<keepBlankLines>false</keepBlankLines>
<lineSeparator>\n</lineSeparator>
<nrOfIndentSpace>2</nrOfIndentSpace>
<predefinedSortOrder>recommended_2008_06</predefinedSortOrder>
<sortDependencies>scope,groupId,artifactId</sortDependencies>
<sortProperties>true</sortProperties>
<spaceBeforeCloseEmptyElement>true</spaceBeforeCloseEmptyElement>
<verifyFail>Stop</verifyFail>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.3.0</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<!-- must be same id as in the apache parent pom, to override the version -->
<id>enforce-maven-version</id>
<id>analyze</id>
<goals>
<goal>enforce</goal>
<goal>analyze-only</goal>
</goals>
<phase>validate</phase>
<configuration>
<rules>
<requireMavenVersion>
<version>[3.5.0,)</version>
</requireMavenVersion>
<requireJavaVersion>
<version>[11,)</version>
</requireJavaVersion>
</rules>
<failOnWarning>true</failOnWarning>
<ignoredUsedUndeclaredDependencies />
<ignoredUnusedDeclaredDependencies>
<runtime>org.apache.logging.log4j:log4j-slf4j-impl</runtime>
</ignoredUnusedDeclaredDependencies>
</configuration>
</execution>
</executions>
@@ -130,15 +143,15 @@
<!-- This was added to ensure project only uses public API -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.1.0</version>
<version>3.1.2</version>
<configuration>
<configLocation>contrib/checkstyle.xml</configLocation>
</configuration>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>8.29</version>
<version>9.3</version>
</dependency>
</dependencies>
<executions>
@@ -153,10 +166,10 @@
<plugin>
<groupId>net.revelc.code</groupId>
<artifactId>impsort-maven-plugin</artifactId>
<version>1.3.0</version>
<version>1.6.2</version>
<configuration>
<removeUnused>true</removeUnused>
<groups>java.,javax.,org.,com.</groups>
<groups>java.,javax.,jakarta.,org.,com.</groups>
</configuration>
<executions>
<execution>
@@ -191,19 +204,16 @@
<plugin>
<groupId>net.revelc.code.formatter</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>2.10.0</version>
<version>2.18.0</version>
<configuration>
<configFile>${eclipseFormatterStyle}</configFile>
<compilerCompliance>${maven.compiler.source}</compilerCompliance>
<compilerSource>${maven.compiler.source}</compilerSource>
<compilerTargetPlatform>${maven.compiler.target}</compilerTargetPlatform>
<lineEnding>LF</lineEnding>
<overrideConfigCompilerVersion>true</overrideConfigCompilerVersion>
<skipJsFormatting>true</skipJsFormatting>
<skipCssFormatting>true</skipCssFormatting>
<skipHtmlFormatting>true</skipHtmlFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
<skipJsFormatting>true</skipJsFormatting>
<skipJsonFormatting>true</skipJsonFormatting>
<skipCssFormatting>true</skipCssFormatting>
<skipXmlFormatting>true</skipXmlFormatting>
</configuration>
<executions>
<execution>
@@ -25,6 +25,7 @@
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -38,6 +39,8 @@
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class CopyPlus5K {

0 comments on commit 88ee1ec

Please sign in to comment.