From 6a2c3d58df27efaa10bc00c7c8a76730ef8bde64 Mon Sep 17 00:00:00 2001 From: Tsz Wo Nicholas Sze Date: Tue, 7 Aug 2018 14:37:05 -0700 Subject: [PATCH] RATIS-288. Pom cleanup/simplification. Contributed by Josh Elser --- .gitignore | 2 - BUILDING.md | 42 +-- pom.xml | 30 ++- ratis-assembly/pom.xml | 1 - ratis-client/pom.xml | 1 - ratis-common/pom.xml | 3 +- ratis-examples/pom.xml | 38 +-- ratis-hadoop-shaded/.gitignore | 2 - ratis-hadoop-shaded/pom.xml | 307 +++++---------------- ratis-proto-shaded/.gitignore | 2 - ratis-proto-shaded/pom.xml | 478 ++++++--------------------------- 11 files changed, 203 insertions(+), 703 deletions(-) delete mode 100644 ratis-hadoop-shaded/.gitignore delete mode 100644 ratis-proto-shaded/.gitignore diff --git a/.gitignore b/.gitignore index 2608160e33..bab2050b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,11 +9,9 @@ *.suo *.vcxproj.user .idea -.svn .classpath .project .settings target build patchprocess -dependency-reduced-pom.xml diff --git a/BUILDING.md b/BUILDING.md index e019eea495..56b65560e9 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -19,7 +19,7 @@ It is required to have Maven 3.3.9 or later. Apache Ratis is written in Java 8. Therefore, it as well requires Java 8 or later. -Project could be built as a usual maven project: +Project could be built as a usual Maven project: ``` $ mvn clean package -DskipTests @@ -34,34 +34,39 @@ We shade protos, protobuf and other libraries such as Netty, gRPC, Guava and Had so that applications using Ratis may use protobuf and other libraries with versions different from the versions used here. -Ratis requires the shaded sources for compilation. -The generated sources are stored in the following directories. -``` -ratis-proto-shaded/src/main/java/ -ratis-hadoop-shaded/src/main/java/ -``` -They are not checked-in to git though. +_Note: RATIS-288 changes how the shaded artifacts are generated, removing them from the +source tree. Developers with local checkouts prior to this change will need to manually +remove the directories `ratis-proto-shaded/src/main/java` and +`ratis-hadoop-shaded/src/main/java`._ -By default protobuf compilation and shading are triggered at every build. To make more faster the -additional builds, you can turn them off: +By default, protobuf compilation and shaded jar creation are executed for every build. + +For developers who wish to skip protobuf generation and shaded jar creation because they +are aware that they have not been modified, they can be disabled with the `skipShade` property. ``` $ mvn package -DskipTests -DskipShade ``` -During the clean lifecycle all the shaded classes are also deleted. You can clean the -compiled files but keep the shaded classes with the following command: - +When the `skipShade` property is given, Maven will inspect your local Maven repository for +the most recent version of `ratis-proto-shaded` (or `ratis-hadoop-shaded`), reaching out to +Maven central when you have no local copy. You may need to run a `mvn install` prior to +attempting to use the `skipShade` property to ensure that you have a version of the artifact +available for your use. ``` -$ mvn clean -DskipCleanShade +$ mvn install -DskipTests ``` -Then, the unit tests can be run by the following commands. +For developers familiar with the `skipCleanShade` option, this is no longer necessary. Maven's +local repository is acting as a cache instead of the current working copy of your repository. +`mvn clean` can be used to safely clean all temporary build files, without impacting your +use of the `skipShade` option. +Unit tests can also be executed with the `skipShade` option: ``` -$ mvn -DskipShade test +$ mvn package -DskipShade ``` -## What are shaded? +## What packages are shaded? | Original packages | Shaded packages | | ------------------------------------|--------------------------------------------------------------| @@ -72,4 +77,5 @@ $ mvn -DskipShade test | `io.netty` | `org.apache.ratis.shaded.io.netty` | | `org.apache.hadoop.ipc.protobuf` | `org.apache.ratis.shaded.org.apache.hadoop.ipc.protobuf` | -The protos defined in this project are stored in the `org.apache.ratis.shaded.proto` package. +The compiled protocol-buffer definitions in this `ratis-proto-shaded` are stored in the +`org.apache.ratis.shaded.proto` Java package. diff --git a/pom.xml b/pom.xml index 31ac6b2e61..c3fb15ab88 100644 --- a/pom.xml +++ b/pom.xml @@ -41,14 +41,12 @@ - ratis-proto-shaded ratis-common ratis-client ratis-server ratis-grpc ratis-netty - ratis-hadoop-shaded ratis-hadoop ratis-assembly @@ -322,6 +320,12 @@ 1.72 + + io.dropwizard.metrics + metrics-core + 3.2.5 + + junit junit @@ -557,6 +561,13 @@ org.apache.rat apache-rat-plugin ${apache-rat-plugin.version} + + + + + target/* + + org.apache.maven.plugins @@ -775,7 +786,8 @@ apache-rat-plugin - package + + check-licenses check @@ -785,5 +797,17 @@ + + skipShade + + + !skipShade + + + + ratis-proto-shaded + ratis-hadoop-shaded + + diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml index 62a83a78de..126f1abddd 100644 --- a/ratis-assembly/pom.xml +++ b/ratis-assembly/pom.xml @@ -21,7 +21,6 @@ ratis-assembly - org.apache.ratis Apache Ratis Project Assembly pom diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml index 6b3ae2c508..b12cb106c1 100644 --- a/ratis-client/pom.xml +++ b/ratis-client/pom.xml @@ -38,7 +38,6 @@ org.slf4j slf4j-api - compile diff --git a/ratis-common/pom.xml b/ratis-common/pom.xml index ff6ec40606..381007452e 100644 --- a/ratis-common/pom.xml +++ b/ratis-common/pom.xml @@ -25,8 +25,8 @@ - ratis-proto-shaded org.apache.ratis + ratis-proto-shaded provided @@ -53,7 +53,6 @@ io.dropwizard.metrics metrics-core - 3.2.5 diff --git a/ratis-examples/pom.xml b/ratis-examples/pom.xml index 0c638830bd..877abbb846 100644 --- a/ratis-examples/pom.xml +++ b/ratis-examples/pom.xml @@ -142,6 +142,7 @@ shade + ${project.build.directory}/dependency-reduced-pom.xml @@ -181,41 +182,4 @@ - - - clean-shade - - - !skipCleanShade - - - - - - maven-clean-plugin - - - shaded-clean - - clean - - clean - - - - . - - dependency-reduced-pom.xml - - false - - - - - - - - - - diff --git a/ratis-hadoop-shaded/.gitignore b/ratis-hadoop-shaded/.gitignore deleted file mode 100644 index ce50505bf1..0000000000 --- a/ratis-hadoop-shaded/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -src/main/java -dependency-reduced-pom.xml diff --git a/ratis-hadoop-shaded/pom.xml b/ratis-hadoop-shaded/pom.xml index ba62721a35..57f4c6ce6c 100644 --- a/ratis-hadoop-shaded/pom.xml +++ b/ratis-hadoop-shaded/pom.xml @@ -25,45 +25,13 @@ true - - ${project.build.directory}/classes - - src/main/java ${hadoop.version} - - ${shaded.sources.dir} - ${classes.dir} - - org.apache.maven.plugins - maven-compiler-plugin - - - - -Xlint:none - -XDignore.symbol.file - - false - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - org/apache/ratis/shaded/**/* - - - - - org.apache.maven.plugins - maven-source-plugin - maven-assembly-plugin @@ -71,141 +39,106 @@ true - - org.apache.rat - apache-rat-plugin - - - ${shaded.sources.dir}/ - - - - - package - - check - - - - + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + + + + - ratis-proto-shaded - org.apache.ratis - provided + org.apache.hadoop + hadoop-common + ${shaded.hadoop.version} - - com.google.protobuf - protobuf-java - ${shaded.protobuf.version} - true + com.google.errorprone + error_prone_annotations + 2.2.0 - com.google.protobuf.nano - protobuf-javanano - ${shaded.protobuf-javanano.version} - true + com.google.j2objc + j2objc-annotations + 1.3 - - org.apache.hadoop - hadoop-common - ${shaded.hadoop.version} - true + org.codehaus.mojo + animal-sniffer-annotations + 1.16 - - skip-shaded-tests - - - skip-shaded-tests - - - - true - - - - - clean-shade + compile-protobuf + - !skipCleanShade + !skipShade - maven-clean-plugin + maven-jar-plugin - shaded-clean + + + default-jar + process-classes - clean + jar - clean - - - - ${shaded.sources.dir} - false - - - . - - dependency-reduced-pom.xml - - false - - - - - - - - - compile-protobuf - - - - !skipShade - - - - compile-protobuf - - ${profile.id}.${project.artifactId}-${project.version} - - - ${jar.finalName} - org.apache.maven.plugins maven-shade-plugin - process-resources + process-classes shade true true - true + + false + ${project.build.directory}/dependency-reduced-pom.xml com.google.protobuf @@ -227,125 +160,19 @@ - - org.apache.ratis:ratis-hadoop-shaded - - com.google.code.findbugs:jsr305 - com.google.code.gson:gson - com.google.guava:guava - com.google.protobuf.nano:protobuf-javanano - com.google.protobuf:protobuf-java - com.google.re2j:re2j - - com.jamesmurty.utils:java-xmlbuilder - com.jcraft:jsch - com.nimbusds:nimbus-jose-jwt - com.sun.jersey:jersey-core - com.sun.jersey:jersey-json - com.sun.jersey:jersey-server - com.sun.jersey:jersey-servlet - com.sun.xml.bind:jaxb-impl - com.thoughtworks.paranamer:paranamer - - commons-beanutils:commons-beanutils-core - commons-beanutils:commons-beanutils - commons-cli:commons-cli - commons-codec:commons-codec - commons-collections:commons-collections - commons-configuration:commons-configuration - commons-digester:commons-digester - commons-io:commons-io - commons-lang:commons-lang - commons-logging:commons-logging - commons-net:commons-net - - io.netty:netty - javax.activation:activation - javax.servlet.jsp:jsp-api - javax.servlet:servlet-api - javax.ws.rs:jsr311-api - javax.xml.bind:jaxb-api - javax.xml.stream:stax-api - log4j:log4j - net.java.dev.jets3t:jets3t - net.jcip:jcip-annotations - net.minidev:json-smart - - org.apache.avro:avro - org.apache.commons:commons-compress - org.apache.commons:commons-math3 - org.apache.curator:curator-client - org.apache.curator:curator-framework - org.apache.curator:curator-recipes - org.apache.hadoop:hadoop-annotations - org.apache.hadoop:hadoop-auth - org.apache.htrace:htrace-core4 - org.apache.httpcomponents:httpclient - org.apache.httpcomponents:httpcore - org.apache.kerby:kerb-admin - org.apache.kerby:kerb-client - org.apache.kerby:kerb-common - org.apache.kerby:kerb-core - org.apache.kerby:kerb-crypto - org.apache.kerby:kerb-identity - org.apache.kerby:kerb-server - org.apache.kerby:kerb-simplekdc - org.apache.kerby:kerb-util - org.apache.kerby:kerby-asn1 - org.apache.kerby:kerby-config - org.apache.kerby:kerby-pkix - org.apache.kerby:kerby-util - org.apache.zookeeper:zookeeper - - org.checkerframework:checker-compat-qual - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-jaxrs - org.codehaus.jackson:jackson-mapper-asl - org.codehaus.jackson:jackson-xc - org.codehaus.jettison:jettison - org.mortbay.jetty:jetty-sslengine - org.mortbay.jetty:jetty-util - org.mortbay.jetty:jetty - org.slf4j:slf4j-api - org.slf4j:slf4j-log4j12 - org.tukaani:xz - org.xerial.snappy:snappy-java - xmlenc:xmlenc - + + + org.apache.hadoop:hadoop-common + com.google.j2objc:j2objc-annotations + com.google.errorprone:error_prone_annotations + org.codehaus.mojo:animal-sniffer-annotations + - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - process-resources - - unpack - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - shaded-sources - jar - true - ${basedir}/${shaded.sources.dir} - **/*.java - - - - - - diff --git a/ratis-proto-shaded/.gitignore b/ratis-proto-shaded/.gitignore deleted file mode 100644 index ce50505bf1..0000000000 --- a/ratis-proto-shaded/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -src/main/java -dependency-reduced-pom.xml diff --git a/ratis-proto-shaded/pom.xml b/ratis-proto-shaded/pom.xml index 2b203fc466..96daef885e 100644 --- a/ratis-proto-shaded/pom.xml +++ b/ratis-proto-shaded/pom.xml @@ -27,8 +27,6 @@ true ${project.build.directory}/classes - - src/main/java 1.10.0 @@ -82,90 +80,94 @@ true - - org.apache.rat - apache-rat-plugin - - - ${shaded.sources.dir}/ - - - - - package - - check - - - - + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.1.1,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + [1.7,) + + run + + + + + + + + + + + + - - - - io.netty - netty-all - ${shaded.netty.version} - pom - import - - - - com.google.protobuf protobuf-java ${shaded.protobuf.version} - true com.google.protobuf.nano protobuf-javanano ${shaded.protobuf-javanano.version} - true io.grpc grpc-netty ${shaded.grpc.version} - true io.grpc grpc-protobuf ${shaded.grpc.version} - true io.grpc grpc-stub ${shaded.grpc.version} - true io.grpc grpc-context ${shaded.grpc.version} - true com.google.auto auto-common 0.10 - compile - true com.google.auto.service auto-service 1.0-rc4 - compile - true com.google.auto.value @@ -176,7 +178,6 @@ com.google.auto.value auto-value 1.6 - provided com.google.guava @@ -188,19 +189,16 @@ com.google.code.findbugs jsr305 3.0.2 - true com.google.errorprone error_prone_annotations 2.2.0 - true com.google.j2objc j2objc-annotations 1.3 - true @@ -213,230 +211,74 @@ org.codehaus.mojo animal-sniffer-annotations 1.16 - true io.netty netty-all ${shaded.netty.version} - true - - - - io.netty - netty-buffer - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-dns - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-haproxy - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-http - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-http2 - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-memcache - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-mqtt - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-socks - ${shaded.netty.version} - compile - true - - - io.netty - netty-codec-stomp - ${shaded.netty.version} - compile - true - - - io.netty - netty-common - ${shaded.netty.version} - compile - true - - - io.netty - netty-handler - ${shaded.netty.version} - compile - true - - - io.netty - netty-handler-proxy - ${shaded.netty.version} - compile - true - - - io.netty - netty-resolver - ${shaded.netty.version} - compile - true - - - io.netty - netty-resolver-dns - ${shaded.netty.version} - compile - true - - - io.netty - netty-transport - ${shaded.netty.version} - compile - true - - io.netty - netty-transport-rxtx - ${shaded.netty.version} - compile - true - - - io.netty - netty-transport-sctp - ${shaded.netty.version} - compile - true - - - io.netty - netty-transport-udt - ${shaded.netty.version} - compile - true - - - io.netty - netty-example - ${shaded.netty.version} - compile - true - - io.netty netty-tcnative ${netty-tcnative.version} ${os.detected.classifier} - compile - true org.jboss.marshalling jboss-marshalling - compile - true + 1.4.11.Final org.slf4j slf4j-api - true commons-logging commons-logging - true + 1.2 org.apache.logging.log4j log4j-api - true + 2.6.2 log4j log4j - true + 1.2.17 com.jcraft jzlib 1.1.3 - true com.ning compress-lzf 1.0.3 - true net.jpountz.lz4 lz4 1.3.0 - true com.github.jponge lzma-java 1.3 - true org.bouncycastle bcpkix-jdk15on 1.54 - compile - true com.google.code.gson gson 2.8.2 - compile - true @@ -462,84 +304,25 @@ com.fasterxml aalto-xml 1.0.0 - compile - true io.opencensus opencensus-api ${io.opencensus.version} - compile - true io.opencensus opencensus-contrib-grpc-metrics ${io.opencensus.version} - compile - true - - - skip-shaded-tests - - - skip-shaded-tests - - - - true - - - - - clean-shade - - - !skipCleanShade - - - - - - maven-clean-plugin - - - shaded-clean - - clean - - clean - - - - ${shaded.sources.dir} - false - - - . - - dependency-reduced-pom.xml - - false - - - - - - - - - - compile-protobuf @@ -549,21 +332,9 @@ - compile-protobuf - ${project.build.directory}/protoc-generated-sources - ${project.build.directory}/protoc-generated-classes - - ${profile.id}.${project.artifactId}-${project.version} - ${jar.finalName} - org.xolstice.maven.plugins protobuf-maven-plugin @@ -571,7 +342,9 @@ com.google.protobuf:protoc:${shaded.protobuf.version}:exe:${os.detected.classifier} - src/main/java + + ${project.build.directory}/generated-sources + false @@ -580,7 +353,6 @@ generate-sources compile - test-compile @@ -588,7 +360,6 @@ generate-sources compile-custom - test-compile-custom grpc-java @@ -600,24 +371,16 @@ - maven-antrun-plugin + maven-jar-plugin - generate-sources - - - - - - - - - - + + + default-jar + process-classes - run + jar @@ -627,14 +390,16 @@ maven-shade-plugin - process-resources + process-classes shade true true - true + + false + ${project.build.directory}/dependency-reduced-pom.xml com.google.api @@ -686,112 +451,35 @@ io.opencensus org.apache.ratis.shaded.io.opencensus + + com.google.gson + org.apache.ratis.shaded.com.google.gson + - - - io.netty:netty-all - - io/netty/** - - - - - - org.apache.ratis:ratis-proto-shaded - - com.barchart.udt:barchart-udt-bundle - com.github.jponge:lzma-java - - com.google.code.findbugs:jsr305 - com.google.auto:auto-common - com.google.auto.service:auto-service - com.google.auto.value:auto-value-annotations - com.google.errorprone:error_prone_annotations - com.google.code.gson:gson - com.google.j2objc:j2objc-annotations - - com.fasterxml:aalto-xml - com.jcraft:jzlib - com.ning:compress-lzf - com.squareup:javapoet - com.yammer.metrics:metrics-core - commons-logging:commons-logging - - io.netty:netty-buffer - io.netty:netty-codec-dns - io.netty:netty-codec-haproxy - io.netty:netty-codec-http2 - io.netty:netty-codec-http - io.netty:netty-codec-memcache - io.netty:netty-codec-mqtt - io.netty:netty-codec-redis - io.netty:netty-codec-socks - io.netty:netty-codec-stomp - io.netty:netty-codec - io.netty:netty-common - io.netty:netty-example - io.netty:netty-handler-proxy - io.netty:netty-handler - io.netty:netty-resolver-dns - io.netty:netty-resolver - io.netty:netty-tcnative - io.netty:netty-transport-rxtx - io.netty:netty-transport-sctp - io.netty:netty-transport-udt - io.netty:netty-transport - - log4j:log4j - net.jpountz.lz4:lz4 - org.apache.logging.log4j:log4j-api - org.bouncycastle:bcpkix-jdk15on - org.bouncycastle:bcprov-jdk15on - org.checkerframework:checker-compat-qual - org.codehaus.mojo:animal-sniffer-annotations - org.codehaus.woodstox:stax2-api - org.conscrypt:conscrypt-openjdk-uber - org.javassist:javassist - org.jboss.marshalling:jboss-marshalling - org.jctools:jctools-core - org.rxtx:rxtx - org.slf4j:slf4j-api - + + com.google.api.grpc:proto-google-common-protos + com.google.code.gson:gson + com.google.guava:guava + com.google.protobuf.nano:protobuf-javanano + com.google.protobuf:protobuf-java-util + com.google.protobuf:protobuf-java + io.grpc:grpc-context + io.grpc:grpc-core + io.grpc:grpc-netty + io.grpc:grpc-protobuf-lite + io.grpc:grpc-protobuf + io.grpc:grpc-stub + io.netty:netty-all + io.opencensus:opencensus-api + io.opencensus:opencensus-contrib-grpc-metrics + - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - process-resources - - unpack - - - - - ${project.groupId} - ${project.artifactId} - ${project.version} - shaded-sources - jar - true - ${basedir}/${shaded.sources.dir} - **/*.java - - - - - -