-
Notifications
You must be signed in to change notification settings - Fork 491
/
do-component-build
151 lines (134 loc) · 6.06 KB
/
do-component-build
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
######################################################################
# BIGTOP-2288 Please remove this patch when maven repository is updated
# with version of leveldbjni-all library that is not x86 specific
if [ $HOSTTYPE = "powerpc64le" ] ; then
#download the POWER version of leveldb
git clone https://github.com/ibmsoe/leveldb.git
git clone https://github.com/ibmsoe/leveldbjni.git
export SNAPPY_HOME=/usr/lib
export LEVELDB_HOME=`cd leveldb; pwd`
export LEVELDBJNI_HOME=`cd leveldbjni; pwd`
export LIBRARY_PATH=${SNAPPY_HOME}
cd ${LEVELDB_HOME}
export C_INCLUDE_PATH=${LIBRARY_PATH}
export CPLUS_INCLUDE_PATH=${LIBRARY_PATH}
git apply ../leveldbjni/leveldb.patch
make libleveldb.a
#Now use maven to build and update the local maven repository with ppc version of
#leveldbjni.
cd ${LEVELDBJNI_HOME}
mvn clean install -DskipTests -P download -Plinux64,all
cd ..
#cleanup
rm -rf ${LEVELDBJNI_HOME}
rm -rf ${LEVELDB_HOME}
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
-Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-2.5.0/bin/protoc
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.6.1 \
-Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.6.1/bin/protoc
mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.26.0 \
-Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/src/grpc-java-1.26.0/compiler/build/exe/java_plugin/protoc-gen-grpc-java
fi
## BIGTOP-2288
# BIGTOP-3027: Build native library/jars for aarch64
if [ $HOSTTYPE = "aarch64" ] ; then
# download leveldbjni 1.8 from original repo
git clone https://github.com/chirino/leveldb.git
git clone https://github.com/fusesource/leveldbjni.git --branch leveldbjni-1.8 --single-branch
export SNAPPY_HOME=/usr/lib
export LEVELDB_HOME=`cd leveldb; pwd`
export LEVELDBJNI_HOME=`cd leveldbjni; pwd`
export LIBRARY_PATH=${SNAPPY_HOME}
cd ${LEVELDB_HOME}
export C_INCLUDE_PATH=${LIBRARY_PATH}
export CPLUS_INCLUDE_PATH=${LIBRARY_PATH}
# apply ARM64 specific patch
patch -p1 -E << 'EOF'
diff a/port/atomic_pointer.h b/port/atomic_pointer.h
--- a/port/atomic_pointer.h
+++ b/port/atomic_pointer.h
@@ -36,6 +36,8 @@
#define ARCH_CPU_X86_FAMILY 1
#elif defined(__ARMEL__)
#define ARCH_CPU_ARM_FAMILY 1
+#elif defined(__aarch64__)
+#define ARCH_CPU_ARM64_FAMILY 1
#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
#define ARCH_CPU_PPC_FAMILY 1
#endif
@@ -93,6 +95,13 @@ inline void MemoryBarrier() {
}
#define LEVELDB_HAVE_MEMORY_BARRIER
+// ARM64
+#elif defined(ARCH_CPU_ARM64_FAMILY) && defined(__linux__)
+inline void MemoryBarrier() {
+ asm volatile("dmb sy" : : : "memory");
+}
+#define LEVELDB_HAVE_MEMORY_BARRIER
+
// PPC
#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
inline void MemoryBarrier() {
@@ -216,6 +225,7 @@ class AtomicPointer {
#undef LEVELDB_HAVE_MEMORY_BARRIER
#undef ARCH_CPU_X86_FAMILY
#undef ARCH_CPU_ARM_FAMILY
+#undef ARCH_CPU_ARM64_FAMILY
#undef ARCH_CPU_PPC_FAMILY
} // namespace port
EOF
git apply ../leveldbjni/leveldb.patch
make libleveldb.a
# Now use maven to build and update the local maven repository with
# aarch64 version of leveldbjni.
cd ${LEVELDBJNI_HOME}
mvn clean install -DskipTests -P download -Plinux64,all
cd ..
#cleanup
rm -rf ${LEVELDBJNI_HOME}
rm -rf ${LEVELDB_HOME}
fi
## BIGTOP-3027
. `dirname $0`/bigtop.bom
if [ -z "$BUNDLE_SNAPPY" ] ; then
[ -f /usr/lib/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib"
[ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
[ -f /usr/lib/${HOSTTYPE}-linux-gnu/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib/${HOSTTYPE}-linux-gnu"
fi
mkdir build
mkdir build/src
# Build artifacts
MAVEN_OPTS="-Dzookeeper.version=$ZOOKEEPER_VERSION "
MAVEN_OPTS+="-Dleveldbjni.group=org.fusesource.leveldbjni "
MAVEN_OPTS+="-DskipTests -DskipITs "
# Include common Maven Deployment logic
. $(dirname ${0})/maven_deploy.sh
# Build artifacts
mvn $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@"
mvn site site:stage ${MAVEN_OPTS} $@
(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
(cd build/src ; tar --strip-components=1 -xzvf ../../hadoop-dist/target/hadoop-${HADOOP_VERSION}-src.tar.gz)
# Copy generated docs
mkdir -p build/share
cp -r target/staging/hadoop-project build/share/doc
# Create a manifest for hadoop client package
(cd hadoop-client-modules/hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list
# Copy fuse output to the build directory
cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs build/bin
cp -r hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp build/include/