Skip to content

Commit

Permalink
use awk to extract classifier
Browse files Browse the repository at this point in the history
  • Loading branch information
@mengCareers
mengCareers committed Jan 15, 2020
1 parent 2f7b573 commit 4738b80
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 16 deletions.
8 changes: 4 additions & 4 deletions dev/deps/spark-deps-hadoop-2.6
View file
Expand Up @@ -19,7 +19,7 @@ arrow-memory/0.10.0/arrow-memory-0.10.0.jar
arrow-vector/0.10.0/arrow-vector-0.10.0.jar
automaton/1.11-8/automaton-1.11-8.jar
avro-ipc/1.8.2/avro-ipc-1.8.2.jar
avro-mapred/1.8.2/avro-mapred-1.8.2-hadoop2.jar
avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
avro/1.8.2/avro-1.8.2.jar
bonecp/0.8.0.RELEASE/bonecp-0.8.0.RELEASE.jar
breeze-macros_2.11/0.13.2/breeze-macros_2.11-0.13.2.jar
Expand Down Expand Up @@ -141,7 +141,7 @@ logging-interceptor/3.12.0/logging-interceptor-3.12.0.jar
lz4-java/1.4.0/lz4-java-1.4.0.jar
machinist_2.11/0.6.1/machinist_2.11-0.6.1.jar
macro-compat_2.11/1.1.1/macro-compat_2.11-1.1.1.jar
mesos/1.4.0/mesos-1.4.0-shaded-protobuf.jar
mesos/1.4.0/shaded-protobuf/mesos-1.4.0-shaded-protobuf.jar
metrics-core/3.1.5/metrics-core-3.1.5.jar
metrics-graphite/3.1.5/metrics-graphite-3.1.5.jar
metrics-json/3.1.5/metrics-json-3.1.5.jar
Expand All @@ -153,8 +153,8 @@ objenesis/2.5.1/objenesis-2.5.1.jar
okhttp/3.12.0/okhttp-3.12.0.jar
okio/1.15.0/okio-1.15.0.jar
opencsv/2.3/opencsv-2.3.jar
orc-core/1.5.5/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/orc-mapreduce-1.5.5-nohive.jar
orc-core/1.5.5/nohive/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/nohive/orc-mapreduce-1.5.5-nohive.jar
orc-shims/1.5.5/orc-shims-1.5.5.jar
oro/2.0.8/oro-2.0.8.jar
osgi-resource-locator/1.0.1/osgi-resource-locator-1.0.1.jar
Expand Down
8 changes: 4 additions & 4 deletions dev/deps/spark-deps-hadoop-2.7
View file
Expand Up @@ -19,7 +19,7 @@ arrow-memory/0.10.0/arrow-memory-0.10.0.jar
arrow-vector/0.10.0/arrow-vector-0.10.0.jar
automaton/1.11-8/automaton-1.11-8.jar
avro-ipc/1.8.2/avro-ipc-1.8.2.jar
avro-mapred/1.8.2/avro-mapred-1.8.2-hadoop2.jar
avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
avro/1.8.2/avro-1.8.2.jar
bonecp/0.8.0.RELEASE/bonecp-0.8.0.RELEASE.jar
breeze-macros_2.11/0.13.2/breeze-macros_2.11-0.13.2.jar
Expand Down Expand Up @@ -142,7 +142,7 @@ logging-interceptor/3.12.0/logging-interceptor-3.12.0.jar
lz4-java/1.4.0/lz4-java-1.4.0.jar
machinist_2.11/0.6.1/machinist_2.11-0.6.1.jar
macro-compat_2.11/1.1.1/macro-compat_2.11-1.1.1.jar
mesos/1.4.0/mesos-1.4.0-shaded-protobuf.jar
mesos/1.4.0/shaded-protobuf/mesos-1.4.0-shaded-protobuf.jar
metrics-core/3.1.5/metrics-core-3.1.5.jar
metrics-graphite/3.1.5/metrics-graphite-3.1.5.jar
metrics-json/3.1.5/metrics-json-3.1.5.jar
Expand All @@ -154,8 +154,8 @@ objenesis/2.5.1/objenesis-2.5.1.jar
okhttp/3.12.0/okhttp-3.12.0.jar
okio/1.15.0/okio-1.15.0.jar
opencsv/2.3/opencsv-2.3.jar
orc-core/1.5.5/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/orc-mapreduce-1.5.5-nohive.jar
orc-core/1.5.5/nohive/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/nohive/orc-mapreduce-1.5.5-nohive.jar
orc-shims/1.5.5/orc-shims-1.5.5.jar
oro/2.0.8/oro-2.0.8.jar
osgi-resource-locator/1.0.1/osgi-resource-locator-1.0.1.jar
Expand Down
8 changes: 4 additions & 4 deletions dev/deps/spark-deps-hadoop-3.1
View file
Expand Up @@ -17,7 +17,7 @@ arrow-memory/0.10.0/arrow-memory-0.10.0.jar
arrow-vector/0.10.0/arrow-vector-0.10.0.jar
automaton/1.11-8/automaton-1.11-8.jar
avro-ipc/1.8.2/avro-ipc-1.8.2.jar
avro-mapred/1.8.2/avro-mapred-1.8.2-hadoop2.jar
avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar
avro/1.8.2/avro-1.8.2.jar
bonecp/0.8.0.RELEASE/bonecp-0.8.0.RELEASE.jar
breeze-macros_2.11/0.13.2/breeze-macros_2.11-0.13.2.jar
Expand Down Expand Up @@ -158,7 +158,7 @@ logging-interceptor/3.12.0/logging-interceptor-3.12.0.jar
lz4-java/1.4.0/lz4-java-1.4.0.jar
machinist_2.11/0.6.1/machinist_2.11-0.6.1.jar
macro-compat_2.11/1.1.1/macro-compat_2.11-1.1.1.jar
mesos/1.4.0/mesos-1.4.0-shaded-protobuf.jar
mesos/1.4.0/shaded-protobuf/mesos-1.4.0-shaded-protobuf.jar
metrics-core/3.1.5/metrics-core-3.1.5.jar
metrics-graphite/3.1.5/metrics-graphite-3.1.5.jar
metrics-json/3.1.5/metrics-json-3.1.5.jar
Expand All @@ -173,8 +173,8 @@ okhttp/2.7.5/okhttp-2.7.5.jar
okhttp/3.12.0/okhttp-3.12.0.jar
okio/1.15.0/okio-1.15.0.jar
opencsv/2.3/opencsv-2.3.jar
orc-core/1.5.5/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/orc-mapreduce-1.5.5-nohive.jar
orc-core/1.5.5/nohive/orc-core-1.5.5-nohive.jar
orc-mapreduce/1.5.5/nohive/orc-mapreduce-1.5.5-nohive.jar
orc-shims/1.5.5/orc-shims-1.5.5.jar
oro/2.0.8/oro-2.0.8.jar
osgi-resource-locator/1.0.1/osgi-resource-locator-1.0.1.jar
Expand Down
13 changes: 9 additions & 4 deletions dev/test-dependencies.sh
View file
Expand Up @@ -85,10 +85,15 @@ for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do

echo "Generating dependency manifest for $HADOOP_PROFILE"
mkdir -p dev/pr-deps
$MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly \
| grep "Dependencies classpath:" -A 1 \
| tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1,2,3 | rev | sort \
| grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE
$MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly | grep "Dependencies classpath:" -A 1 | tail -n 1 | tr ":" "\n" | awk -F '/' '{
artifact_id=$(NF-2);
version=$(NF-1);
jar_name=$(NF-0);
classifier_start_index=length(artifact_id"-"version"-") + 1;
classifier_end_index=index(jar_name, ".jar") - 1;
classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1);
print artifact_id"/"version (length(classifier) > 0 ? "/" : "") classifier"/"jar_name
}' | sort | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE
done

if [[ $@ == **replace-manifest** ]]; then
Expand Down

0 comments on commit 4738b80

Please sign in to comment.