Skip to content

Commit

Permalink
Reorganize for Makefiles and dockerfiles (#2246)
Browse files Browse the repository at this point in the history
This PR reorganized Makefiles and dockerfiles, targets for the consistency of image and non-image version and ease of management.

Some major changes includes:

- Redesigned the directory strucure under `k8s`, make similar targets appears in the same directory. 
    - Some internal stuffs, which very unlikely would be used and be cared about  by users, such as building wheels, building dataset image go to the `internal` directory
    - All dockerfiles go to the `dockerfiles` directory
    - Building scripts go to the `build_scripts` directory
    - All other scripts go to the `utils` directory
    - Add documentation.
- Unified the make targets in root directory and k8s directory. Ideally, the only difference should be that one would produce local artifacts and the other would produce the image.
- Drop unnecessary make targets - push, clean
- Drop registry fields in developing stage for better readability
- Disable the release of nightly jupyter, dataset and graphscope-store image, it's not yet necessary.
  • Loading branch information
siyuan0322 committed Nov 28, 2022
1 parent 84f449c commit 71d61e3
Show file tree
Hide file tree
Showing 39 changed files with 440 additions and 422 deletions.
27 changes: 18 additions & 9 deletions .github/workflows/build-graphscope-wheels-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
version=$(cat ${GITHUB_WORKSPACE}/VERSION)
if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi

cd ${GITHUB_WORKSPACE}/k8s
cd ${GITHUB_WORKSPACE}/k8s/internal
# build graphscope wheels
sudo -E -u runner make graphscope-py3-package

Expand Down Expand Up @@ -112,11 +112,17 @@ jobs:
tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
cd ${GITHUB_WORKSPACE}/k8s
make graphscope-image
make graphscope

make jupyter-image
# make jupyter-image
cd ..
docker build \
--build-arg CI=${CI} \
-t graphscope/jupyter:${SHORT_SHA} \
-f ./k8s/internal/jupyter.Dockerfile .

make dataset-image
# dataset image doesn't changed, we can just use the latest one
# make dataset-image

- name: Release Nightly Image
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -132,12 +138,13 @@ jobs:
# graphscope image
sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope:${tag}
sudo docker push ${{ env.REGISTRY }}/graphscope:${tag}
# jupyter and dataset doesn't need nightly release yet
# jupyter image
sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
# sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
# sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
# dataset image
sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
sudo docker push ${{ env.REGISTRY }}/dataset:${tag}
# sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
# sudo docker push ${{ env.REGISTRY }}/dataset:${tag}

- name: Extract Tag Name
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
Expand All @@ -160,7 +167,9 @@ jobs:
sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
# dataset image
sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
# Note! dataset image are built mannually just use the latest one.
sudo docker pull ${{ env.REGISTRY }}/dataset:latest
sudo docker tag ${{ env.REGISTRY }}/dataset:latest ${{ env.REGISTRY }}/dataset:${tag}
sudo docker push ${{ env.REGISTRY }}/dataset:${tag}

ubuntu-python-test:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build-graphscope-wheels-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
# build graphscope server wheel
cd ${GITHUB_WORKSPACE}/k8s
cd ${GITHUB_WORKSPACE}/k8s/internal
sudo -E make graphscope-py3-package
# package
Expand Down Expand Up @@ -96,7 +96,7 @@ jobs:
if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
# build graphscope client wheel
cd ${GITHUB_WORKSPACE}/k8s
cd ${GITHUB_WORKSPACE}/k8s/internal
make graphscope-client-py3-package
# package
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ jobs:
export RUSTC_WRAPPER=/usr/local/bin/sccache
sccache --start-server
cd ${GITHUB_WORKSPACE}/k8s
cd ${GITHUB_WORKSPACE}/k8s/internal
# build graphscope wheels
make graphscope-manylinux2014-py3-nodocker
# build client wheels
Expand Down Expand Up @@ -604,8 +604,8 @@ jobs:
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
# build graphscope image
cd ${GITHUB_WORKSPACE}
make graphscope-image
cd ${GITHUB_WORKSPACE}/k8s/internal
make graphscope
cd ${GITHUB_WORKSPACE}/python
pip3 install -r requirements.txt
Expand Down Expand Up @@ -658,8 +658,8 @@ jobs:
tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
cd ${GITHUB_WORKSPACE}
make graphscope-image
cd ${GITHUB_WORKSPACE}/k8s/internal
make graphscope
docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:${SHORT_SHA}
cd ${GITHUB_WORKSPACE}/python
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docker/graphscope-store.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ RUN yum install -y sudo java-1.8.0-openjdk-devel bind-utils \
&& yum clean all \
&& rm -rf /var/cache/yum

COPY k8s/ready_probe.sh /tmp/ready_probe.sh
COPY k8s/utils/ready_probe.sh /tmp/ready_probe.sh
ADD artifacts/groot.tar.gz /usr/local

RUN useradd -m graphscope -u 1001 \
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/gae.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ jobs:
RUN_JAVA_TESTS: ON
run: |
# default install to "/opt/graphscope"
make gae ENABLE_JAVA_SDK=ON BUILD_TEST=ON NUMPROC=1
sudo make gae-install
make analytical-java BUILD_TEST=ON NUMPROC=1
sudo make analytical-java-install
# also make coordinator and client for python test
make client
make coordinator
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/networkx-forward-algo-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ jobs:
- name: Build GAE and coordinator
run: |
pushd ${GITHUB_WORKSPACE}
make gae-install ENABLE_JAVA_SDK=OFF BUILD_TEST=OFF NUMPROC=1
install_prefix=/opt/graphscope
mkdir -p ${install_prefix}
make analytical-install BUILD_TEST=OFF NUMPROC=1 INSTALL_PREFIX=${install_prefix}
# also make coordinator and client for python test
make coordinator && make client
popd
Expand Down
9 changes: 6 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@ jobs:
- name: Build GraphScope Store Image
run: |
cd ${GITHUB_WORKSPACE}
sudo make graphscope-store-image
cd ${GITHUB_WORKSPACE}/k8s
sudo make graphscope-store
- name: Release Nightly Image
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
# if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
# GraphScope-Store doesn't need nightly release yet.
# To save some resources, comment this step out temporarily.
if: false
env:
docker_password: ${{ secrets.DOCKER_PASSWORD }}
docker_username: ${{ secrets.DOCKER_USER }}
Expand Down
133 changes: 65 additions & 68 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
WORKING_DIR := $(dir $(MKFILE_PATH))
GAE_DIR := $(WORKING_DIR)/analytical_engine
GIE_DIR := $(WORKING_DIR)/interactive_engine
GLE_DIR := $(WORKING_DIR)/learning_engine/graph-learn
GAE_BUILD_DIR := $(GAE_DIR)/build
GLE_BUILD_DIR := $(GLE_DIR)/graphlearn/cmake-build
ANALYTICAL_DIR := $(WORKING_DIR)/analytical_engine
INTERACTIVE_DIR := $(WORKING_DIR)/interactive_engine
LEARNING_DIR := $(WORKING_DIR)/learning_engine/graph-learn
ANALYTICAL_BUILD_DIR := $(ANALYTICAL_DIR)/build
LEARNING_BUILD_DIR := $(LEARNING_DIR)/graphlearn/cmake-build
CLIENT_DIR := $(WORKING_DIR)/python
COORDINATOR_DIR := $(WORKING_DIR)/coordinator
K8S_DIR := $(WORKING_DIR)/k8s
Expand All @@ -14,14 +14,12 @@ VERSION ?= 0.18.0

BUILD_TYPE ?= release

# GAE build options
# analytical engine build options
NETWORKX ?= ON

# testing build option
BUILD_TEST ?= OFF

# build java sdk option
ENABLE_JAVA_SDK ?= ON

# INSTALL_PREFIX is environment variable, but if it is not set, then set default value
ifeq ($(INSTALL_PREFIX),)
Expand All @@ -42,12 +40,10 @@ endif
## Common
.PHONY: all graphscope install clean

# all: graphscope
# graphscope: gle client coordinator gae gie
all: gle client coordinator gae gie
all: learning client coordinator analytical interactive
graphscope: all

install: gae-install gie-install gle-install client coordinator
install: analytical-install interactive-install learning-install client coordinator
# client
pip3 install --user --editable $(CLIENT_DIR)
rm -rf $(CLIENT_DIR)/*.egg-info
Expand All @@ -59,23 +55,24 @@ install: gae-install gie-install gle-install client coordinator
echo "export GRAPHSCOPE_HOME=$(INSTALL_PREFIX)"

clean:
rm -rf $(GAE_BUILD_DIR) $(GAE_DIR)/proto
cd $(GAE_DIR)/java && mvn clean
rm -rf $(ANALYTICAL_BUILD_DIR) $(ANALYTICAL_DIR)/proto
cd $(ANALYTICAL_DIR)/java && mvn clean

cd $(GIE_DIR) && mvn clean
cd $(INTERACTIVE_DIR) && mvn clean
# TODO: use maven clean to clean ir target
rm -rf $(GIE_DIR)/executor/ir/target
rm -rf $(INTERACTIVE_DIR)/executor/ir/target

rm -rf $(GLE_BUILD_DIR) $(GLE_DIR)/proto/*.h $(GLE_DIR)/proto/*.cc
rm -rf $(LEARNING_BUILD_DIR) $(LEARNING_DIR)/proto/*.h $(LEARNING_DIR)/proto/*.cc

cd $(CLIENT_DIR) && python3 setup.py clean --all

cd $(COORDINATOR_DIR) && python3 setup.py clean --all

## Modules
.PHONY: client coordinator gae gie gle
.PHONY: client coordinator analytical interactive learning
.PHONY: analytical-java

client: gle
client: learning
cd $(CLIENT_DIR) && \
pip3 install -r requirements.txt -r requirements-dev.txt --user && \
python3 setup.py build_ext --inplace --user
Expand All @@ -85,63 +82,81 @@ coordinator: client
pip3 install -r requirements.txt -r requirements-dev.txt --user && \
python3 setup.py build_builtin

.PHONY: gae-install gie-install gle-install
# We deliberately make $(ENGINE) depends on a file, and $(ENGINE)-install depends on $(ENGINE),
# so that when we execute `make $(ENGINE)-install` after `make $(ENGINE)`, it will not
# rebuild $(ENGINE) from scratch.
# If we doesn't make gxe depends on a file (as a PHONY), then make will never
# know if $(ENGINE) is up-to-date or not, so it will always rebuild gxe.
# Note: `$(ENGINE)` stands for `analytical`, `interactive` and `learning`.

gae-install: gae
mkdir -p $(INSTALL_PREFIX)
$(MAKE) -C $(GAE_BUILD_DIR) install
install $(K8S_DIR)/kube_ssh $(INSTALL_PREFIX)/bin/
.PHONY: analytical-install interactive-install learning-install
.PHONY: analytical-java-install

analytical-install: analytical
$(MAKE) -C $(ANALYTICAL_BUILD_DIR) install
install -d $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake
if [ -d "${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical" ]; then \
install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/*.cmake $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical; \
install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/cmake/* $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake; \
fi

analytical: $(ANALYTICAL_BUILD_DIR)/grape_engine

$(ANALYTICAL_BUILD_DIR)/grape_engine:
mkdir -p $(ANALYTICAL_BUILD_DIR) && \
cd $(ANALYTICAL_BUILD_DIR) && \
cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
-DNETWORKX=$(NETWORKX) \
-DBUILD_TESTS=${BUILD_TEST} \
-DENABLE_JAVA_SDK=OFF .. && \
$(MAKE) -j$(NUMPROC)

analytical-java-install: analytical-java
$(MAKE) -C $(ANALYTICAL_BUILD_DIR) install
install -d $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake
if [ -d "${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical" ]; then \
install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/*.cmake $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical; \
install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/cmake/* $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake; \
fi

gae: $(GAE_BUILD_DIR)/grape_engine
analytical-java: $(ANALYTICAL_BUILD_DIR)/graphx_runner

$(GAE_BUILD_DIR)/grape_engine:
mkdir -p $(GAE_BUILD_DIR) && \
cd $(GAE_BUILD_DIR) && \
$(ANALYTICAL_BUILD_DIR)/graphx_runner:
mkdir -p $(ANALYTICAL_BUILD_DIR) && \
cd $(ANALYTICAL_BUILD_DIR) && \
cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
-DNETWORKX=$(NETWORKX) \
-DBUILD_TESTS=${BUILD_TEST} \
-DENABLE_JAVA_SDK=${ENABLE_JAVA_SDK} .. && \
-DENABLE_JAVA_SDK=ON .. && \
$(MAKE) -j$(NUMPROC)

gie-install: gie
interactive-install: interactive
mkdir -p $(INSTALL_PREFIX)
tar -xf $(GIE_DIR)/assembly/target/graphscope.tar.gz --strip-components 1 -C $(INSTALL_PREFIX)
gie: $(GIE_DIR)/assembly/target/graphscope.tar.gz
tar -xf $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz --strip-components 1 -C $(INSTALL_PREFIX)
interactive: $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz

$(GIE_DIR)/assembly/target/graphscope.tar.gz:
# frontend/executor
cd $(GIE_DIR) && \
$(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz:
cd $(INTERACTIVE_DIR) && \
mvn package -DskipTests -Drust.compile.mode=$(BUILD_TYPE) -P graphscope,graphscope-assembly --quiet

gle-install: gle
learning-install: learning
mkdir -p $(INSTALL_PREFIX)
$(MAKE) -C $(GLE_BUILD_DIR) install
gle: $(GLE_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX)
$(MAKE) -C $(LEARNING_BUILD_DIR) install
learning: $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX)

$(GLE_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):
$(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):
git submodule update --init
cd $(GLE_DIR) && git submodule update --init third_party/pybind11
mkdir -p $(GLE_BUILD_DIR)
cd $(GLE_BUILD_DIR) && \
cd $(LEARNING_DIR) && git submodule update --init third_party/pybind11
mkdir -p $(LEARNING_BUILD_DIR)
cd $(LEARNING_BUILD_DIR) && \
cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
-DKNN=OFF \
-DWITH_VINEYARD=ON \
-DTESTING=${BUILD_TEST} .. && \
$(MAKE) -j$(NUMPROC)

## wheels
.PHONY: graphscope-py3-package graphscope-client-py3-package prepare-client graphscope-docs

graphscope-py3-package:
$(MAKE) -C $(K8S_DIR) graphscope-py3-package

graphscope-client-py3-package:
$(MAKE) -C $(K8S_DIR) graphscope-client-py3-package
.PHONY: prepare-client graphscope-docs

prepare-client:
cd $(CLIENT_DIR) && \
Expand All @@ -153,25 +168,6 @@ graphscope-docs: prepare-client
$(MAKE) -C $(DOCS_DIR)/ html


## Images
.PHONY: graphscope-image jupyter-image dataset-image graphscope-store-image push

graphscope-image:
$(MAKE) -C $(K8S_DIR) graphscope-image VERSION=$(VERSION)

jupyter-image:
$(MAKE) -C $(K8S_DIR) jupyter-image VERSION=$(VERSION)

dataset-image:
$(MAKE) -C $(K8S_DIR) dataset-image VERSION=$(VERSION)

graphscope-store-image:
$(MAKE) -C $(K8S_DIR) graphscope-store-image VERSION=$(VERSION)

push:
$(MAKE) -C $(K8S_DIR) push


## Tests
.PHONY: test unittest minitest k8stest

Expand All @@ -190,3 +186,4 @@ k8stest:
pip3 install tensorflow==2.5.2 "pandas<1.5.0"
cd $(CLIENT_DIR) && \
python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml --cov-report=term -s -v ./graphscope/tests/kubernetes

Loading

0 comments on commit 71d61e3

Please sign in to comment.