/
Makefile
222 lines (197 loc) · 8.08 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
WORKING_DIR := $(dir $(MKFILE_PATH))
SHORT_SHA := $(shell git rev-parse --short HEAD)
ifeq ("$(DOCKER_ORG)","")
$(warning WARNING: No docker user found, using graphscope)
DOCKER_ORG = graphscope
endif
.PHONY: submodule gsruntime gsvineyard graphscope push clean
ifeq ($(REGISTRY),)
REGISTRY := registry.cn-hongkong.aliyuncs.com
endif
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S), Linux)
XARGS_EMPTY_FLAG := --no-run-if-empty
endif
# Default null
TARGET_SERVER_PACKAGE :=
TARGET_CLIENT_PACKAGE :=
ifeq ($(UNAME_S), Linux)
TARGET_SERVER_PACKAGE := graphscope-manylinux2014-py3
TARGET_CLIENT_PACKAGE := graphscope-client-manylinux2014-py3
else
TARGET_SERVER_PACKAGE := graphscope-darwin-py3
TARGET_CLIENT_PACKAGE := graphscope-client-darwin-py3
endif
VERSION ?= latest
VINEYARD_VERSION ?= v0.4.1
PROFILE ?= release
BUILD_PROGRESS = auto
submodule:
cd ../ && git submodule update --init
gsruntime-image:
docker build --progress=$(BUILD_PROGRESS) \
-t ${REGISTRY}/$(DOCKER_ORG)/graphscope-runtime:${VERSION} \
--network=host \
-f $(WORKING_DIR)/manylinux2014.Dockerfile .
gsvineyard-image:
docker build --progress=$(BUILD_PROGRESS) \
-t ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VERSION} \
-f $(WORKING_DIR)/gsvineyard.Dockerfile .
graphscope-image: submodule
docker build --progress=$(BUILD_PROGRESS) \
--build-arg CI=${CI} \
-t graphscope/graphscope:${SHORT_SHA} \
--network=host \
-f $(WORKING_DIR)/graphscope.Dockerfile ..
graphscope-dev-image: submodule
docker build --progress=$(BUILD_PROGRESS) \
-t graphscope/graphscope:${SHORT_SHA} \
--network=host \
-f $(WORKING_DIR)/graphscope-dev.Dockerfile ..
jupyter-image:
docker build --progress=$(BUILD_PROGRESS) \
--build-arg CI=${CI} \
-t graphscope/jupyter:${SHORT_SHA} \
--network=host \
-f $(WORKING_DIR)/jupyter.Dockerfile ..
graphscope-store-image:
docker pull ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION}
docker build --progress=$(BUILD_PROGRESS) \
--build-arg profile=${PROFILE} \
-t graphscope/graphscope-store:${SHORT_SHA} \
--network=host \
-f $(WORKING_DIR)/graphscope-store.Dockerfile ..
graphscope-jupyter-manylinux2014-py3:
docker run --rm -it -v $(WORKING_DIR)/..:/work \
${REGISTRY}/$(DOCKER_ORG)/graphscope-runtime:${VERSION} \
bash -c 'pip3 install graphscope-client -U && \
cd /work/python/jupyter/graphscope && \
python3 setup.py bdist_wheel'
graphscope-py3-package: $(TARGET_SERVER_PACKAGE)
graphscope-client-py3-package: $(TARGET_CLIENT_PACKAGE)
graphscope-darwin-py3:
sudo cp -rf /opt/vineyard/* /usr/local/
# Avoid copy vineyard library with the same basename by:
# 1) reinstall vineyard to /usr/local and
# 2) rm /opt/vineyard/lib
cd /tmp && git clone -b ${VINEYARD_VERSION} https://github.com/v6d-io/v6d.git --depth=1 && \
cd v6d && git submodule update --init && \
mkdir -p build && cd build && \
cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local && \
sudo make install -j`nproc` && \
sudo rm -fr /opt/vineyard/lib
sudo rm -fr /tmp/v6d /tmp/libgrape-lite
# build graphscope
cd $(WORKING_DIR)/../ && \
make install && \
python3 $(WORKING_DIR)/precompile.py
# build and delocate wheel
cd $(WORKING_DIR)/../coordinator && \
export WITH_EXTRA_DATA=ON && \
rm -fr build dist/*.whl || true && \
sudo strip -s $(WORKING_DIR)/../analytical_engine/exported_symbols_osx.lds /opt/graphscope/bin/grape_engine && \
sudo strip /opt/graphscope/bin/executor && \
sudo strip /opt/graphscope/bin/gaia_executor && \
export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \
package_name=gs-lib python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-jython python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
rm -fr build && \
package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
pip3 install delocate && \
for wheel in `ls dist/*.whl`; do \
delocate-listdeps -a -d $$wheel; \
delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \
done
graphscope-manylinux2014-py3-nodocker:
cd $(WORKING_DIR)/../ && \
make install && \
python3 $(WORKING_DIR)/precompile.py && \
export WITH_EXTRA_DATA=ON && \
cd $(WORKING_DIR)/../coordinator && \
rm -fr build dist/*.whl && \
sudo strip /opt/graphscope/bin/grape_engine && \
sudo strip /opt/graphscope/bin/executor && \
sudo strip /opt/graphscope/bin/gaia_executor && \
sudo strip /opt/graphscope/lib/*.so && \
strip /tmp/gs/builtin/*/*.so && \
package_name=gs-lib python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-include python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-apps python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-engine python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-jython python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=gs-coordinator python3 setup.py bdist_wheel && \
rm -fr build && \
package_name=graphscope python3 setup.py bdist_wheel && \
cd dist && \
for wheel in `ls ./*.whl`; do \
auditwheel repair $$wheel --plat=manylinux2014_x86_64 && rm $$wheel; \
done
graphscope-manylinux2014-py3:
docker pull ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION}
docker run --rm -v $(WORKING_DIR)/..:/work \
${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION} \
bash -c 'source ~/.bashrc && \
cd /work/k8s && \
make graphscope-manylinux2014-py3-nodocker'
graphscope-client-manylinux2014-py3-nodocker:
cd $(WORKING_DIR)/../learning_engine/graph-learn && \
git submodule update --init third_party/pybind11 && \
rm -rf cmake-build && \
mkdir -p cmake-build && \
cd cmake-build && \
cmake -DWITH_VINEYARD=ON .. && \
make graphlearn_shared -j`nproc` && \
export LD_LIBRARY_PATH=$(WORKING_DIR)/../learning_engine/graph-learn/built/lib:$$LD_LIBRARY_PATH && \
for py in cp37-cp37m cp38-cp38 cp39-cp39 ; do \
cd $(WORKING_DIR)/../python; \
export PATH=/opt/python/$$py/bin:$$PATH; \
pip3 install -U pip numpy==1.18.5 auditwheel==5.0.0 grpcio grpcio_tools wheel ; \
rm -rf build; \
rm -rf dist/*.whl; \
python3 setup.py bdist_wheel; \
cd dist; \
auditwheel repair ./*.whl; \
done
graphscope-client-darwin-py3:
cd $(WORKING_DIR)/../learning_engine/graph-learn && \
git submodule update --init third_party/pybind11 && \
rm -rf cmake-build && \
mkdir -p cmake-build && \
cd cmake-build && \
cmake -DWITH_VINEYARD=ON -DTESTING=OFF .. && \
make graphlearn_shared -j`nproc` && \
export DYLD_LIBRARY_PATH=$(WORKING_DIR)/../learning_engine/graph-learn/built/lib:/usr/local/lib:$$DYLD_LIBRARY_PATH && \
cd $(WORKING_DIR)/../python && \
pip3 install -U pip numpy==1.18.5 delocate grpcio grpcio_tools wheel && \
rm -fr build dist/*.whl || true && \
python3 setup.py bdist_wheel --plat=macosx_10_9_x86_64 && \
for wheel in `ls dist/*.whl`; do \
delocate-wheel -w dist/wheelhouse -v $$wheel && rm $$wheel; \
done
graphscope-client-manylinux2014-py3:
docker pull ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION}
docker run --rm -v $(WORKING_DIR)/..:/work \
${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION} \
bash -c 'cd /work/k8s && make graphscope-client-manylinux2014-py3-nodocker'
push:
docker push ${REGISTRY}/$(DOCKER_ORG)/graphscope-runtime:${VERSION}
docker push ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VERSION}
docker push ${REGISTRY}/$(DOCKER_ORG)/graphscope:${VERSION}
clean:
docker ps -qa | xargs $(XARGS_EMPTY_FLAG) docker rm -f
docker images -f "dangling=true" -q | xargs $(XARGS_EMPTY_FLAG) docker rmi -f