Reorganize for Makefiles and dockerfiles (#2246)

This PR reorganized Makefiles and dockerfiles, targets for the consistency of image and non-image version and ease of management. Some major changes includes: - Redesigned the directory strucure under `k8s`, make similar targets appears in the same directory. - Some internal stuffs, which very unlikely would be used and be cared about by users, such as building wheels, building dataset image go to the `internal` directory - All dockerfiles go to the `dockerfiles` directory - Building scripts go to the `build_scripts` directory - All other scripts go to the `utils` directory - Add documentation. - Unified the make targets in root directory and k8s directory. Ideally, the only difference should be that one would produce local artifacts and the other would produce the image. - Drop unnecessary make targets - push, clean - Drop registry fields in developing stage for better readability - Disable the release of nightly jupyter, dataset and graphscope-store image, it's not yet necessary.
alibaba · Nov 28, 2022 · 71d61e3 · 71d61e3
1 parent 84f449c
commit 71d61e3
Show file tree

Hide file tree

Showing 39 changed files with 440 additions and 422 deletions.
diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml
@@ -42,7 +42,7 @@ jobs:
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
         if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
 
-        cd ${GITHUB_WORKSPACE}/k8s
+        cd ${GITHUB_WORKSPACE}/k8s/internal
         # build graphscope wheels
         sudo -E -u runner make graphscope-py3-package
 
@@ -112,11 +112,17 @@ jobs:
         tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
         tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
         cd ${GITHUB_WORKSPACE}/k8s
-        make graphscope-image
+        make graphscope
 
-        make jupyter-image
+        # make jupyter-image
+        cd ..
+        docker build \
+				 --build-arg CI=${CI} \
+				 -t graphscope/jupyter:${SHORT_SHA} \
+				 -f ./k8s/internal/jupyter.Dockerfile .
 
-        make dataset-image
+        # dataset image doesn't changed, we can just use the latest one
+        # make dataset-image
 
     - name: Release Nightly Image
       if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
@@ -132,12 +138,13 @@ jobs:
         # graphscope image
         sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.REGISTRY }}/graphscope:${tag}
         sudo docker push ${{ env.REGISTRY }}/graphscope:${tag}
+        # jupyter and dataset doesn't need nightly release yet
         # jupyter image
-        sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
-        sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
+        # sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
+        # sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
         # dataset image
-        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
-        sudo docker push ${{ env.REGISTRY }}/dataset:${tag}
+        # sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
+        # sudo docker push ${{ env.REGISTRY }}/dataset:${tag}
 
     - name: Extract Tag Name
       if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
@@ -160,7 +167,9 @@ jobs:
         sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.REGISTRY }}/jupyter:${tag}
         sudo docker push ${{ env.REGISTRY }}/jupyter:${tag}
         # dataset image
-        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.REGISTRY }}/dataset:${tag}
+        # Note! dataset image are built mannually just use the latest one.
+        sudo docker pull ${{ env.REGISTRY }}/dataset:latest
+        sudo docker tag ${{ env.REGISTRY }}/dataset:latest ${{ env.REGISTRY }}/dataset:${tag}
         sudo docker push ${{ env.REGISTRY }}/dataset:${tag}
 
   ubuntu-python-test:

diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml
@@ -45,7 +45,7 @@ jobs:
         if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
 
         # build graphscope server wheel
-        cd ${GITHUB_WORKSPACE}/k8s
+        cd ${GITHUB_WORKSPACE}/k8s/internal
         sudo -E make graphscope-py3-package
 
         # package
@@ -96,7 +96,7 @@ jobs:
         if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi
 
         # build graphscope client wheel
-        cd ${GITHUB_WORKSPACE}/k8s
+        cd ${GITHUB_WORKSPACE}/k8s/internal
         make graphscope-client-py3-package
 
         # package

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -249,7 +249,7 @@ jobs:
         export RUSTC_WRAPPER=/usr/local/bin/sccache
         sccache --start-server
 
-        cd ${GITHUB_WORKSPACE}/k8s
+        cd ${GITHUB_WORKSPACE}/k8s/internal
         # build graphscope wheels
         make graphscope-manylinux2014-py3-nodocker
         # build client wheels
@@ -604,8 +604,8 @@ jobs:
           tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
 
           # build graphscope image
-          cd ${GITHUB_WORKSPACE}
-          make graphscope-image
+          cd ${GITHUB_WORKSPACE}/k8s/internal
+          make graphscope
 
           cd ${GITHUB_WORKSPACE}/python
           pip3 install -r requirements.txt
@@ -658,8 +658,8 @@ jobs:
           tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
           tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
 
-          cd ${GITHUB_WORKSPACE}
-          make graphscope-image
+          cd ${GITHUB_WORKSPACE}/k8s/internal
+          make graphscope
           docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:${SHORT_SHA}
 
           cd ${GITHUB_WORKSPACE}/python

diff --git a/.github/workflows/docker/graphscope-store.Dockerfile b/.github/workflows/docker/graphscope-store.Dockerfile
@@ -4,7 +4,7 @@ RUN yum install -y sudo java-1.8.0-openjdk-devel bind-utils \
     && yum clean all \
     && rm -rf /var/cache/yum
 
-COPY k8s/ready_probe.sh /tmp/ready_probe.sh
+COPY k8s/utils/ready_probe.sh /tmp/ready_probe.sh
 ADD artifacts/groot.tar.gz /usr/local
 
 RUN useradd -m graphscope -u 1001 \

diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml
@@ -57,8 +57,8 @@ jobs:
         RUN_JAVA_TESTS: ON
       run: |
         # default install to "/opt/graphscope"
-        make gae ENABLE_JAVA_SDK=ON BUILD_TEST=ON NUMPROC=1
-        sudo make gae-install
+        make analytical-java BUILD_TEST=ON NUMPROC=1
+        sudo make analytical-java-install
         # also make coordinator and client for python test
         make client
         make coordinator

diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml
@@ -26,7 +26,9 @@ jobs:
     - name: Build GAE and coordinator
       run: |
         pushd ${GITHUB_WORKSPACE}
-        make gae-install ENABLE_JAVA_SDK=OFF BUILD_TEST=OFF NUMPROC=1
+        install_prefix=/opt/graphscope
+        mkdir -p ${install_prefix}
+        make analytical-install BUILD_TEST=OFF NUMPROC=1 INSTALL_PREFIX=${install_prefix}
         # also make coordinator and client for python test
         make coordinator && make client
         popd

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -32,11 +32,14 @@ jobs:
 
     - name: Build GraphScope Store Image
       run: |
-        cd ${GITHUB_WORKSPACE}
-        sudo make graphscope-store-image
+        cd ${GITHUB_WORKSPACE}/k8s
+        sudo make graphscope-store
 
     - name: Release Nightly Image
-      if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      # GraphScope-Store doesn't need nightly release yet.
+      # To save some resources, comment this step out temporarily.
+      if: false
       env:
         docker_password: ${{ secrets.DOCKER_PASSWORD }}
         docker_username: ${{ secrets.DOCKER_USER }}

diff --git a/Makefile b/Makefile
@@ -1,10 +1,10 @@
 MKFILE_PATH				:= $(abspath $(lastword $(MAKEFILE_LIST)))
 WORKING_DIR				:= $(dir $(MKFILE_PATH))
-GAE_DIR					:= $(WORKING_DIR)/analytical_engine
-GIE_DIR					:= $(WORKING_DIR)/interactive_engine
-GLE_DIR					:= $(WORKING_DIR)/learning_engine/graph-learn
-GAE_BUILD_DIR			:= $(GAE_DIR)/build
-GLE_BUILD_DIR			:= $(GLE_DIR)/graphlearn/cmake-build
+ANALYTICAL_DIR			:= $(WORKING_DIR)/analytical_engine
+INTERACTIVE_DIR			:= $(WORKING_DIR)/interactive_engine
+LEARNING_DIR			:= $(WORKING_DIR)/learning_engine/graph-learn
+ANALYTICAL_BUILD_DIR	:= $(ANALYTICAL_DIR)/build
+LEARNING_BUILD_DIR		:= $(LEARNING_DIR)/graphlearn/cmake-build
 CLIENT_DIR				:= $(WORKING_DIR)/python
 COORDINATOR_DIR			:= $(WORKING_DIR)/coordinator
 K8S_DIR					:= $(WORKING_DIR)/k8s
@@ -14,14 +14,12 @@ VERSION					?= 0.18.0
 
 BUILD_TYPE				?= release
 
-# GAE build options
+# analytical engine build options
 NETWORKX				?= ON
 
 # testing build option
 BUILD_TEST				?= OFF
 
-# build java sdk option
-ENABLE_JAVA_SDK			?= ON
 
 # INSTALL_PREFIX is environment variable, but if it is not set, then set default value
 ifeq ($(INSTALL_PREFIX),)
@@ -42,12 +40,10 @@ endif
 ## Common
 .PHONY: all graphscope install clean
 
-# all: graphscope
-# graphscope: gle client coordinator gae gie
-all: gle client coordinator gae gie
+all: learning client coordinator analytical interactive
 graphscope: all
 
-install: gae-install gie-install gle-install client coordinator
+install: analytical-install interactive-install learning-install client coordinator
     # client
 	pip3 install --user --editable $(CLIENT_DIR)
 	rm -rf $(CLIENT_DIR)/*.egg-info
@@ -59,23 +55,24 @@ install: gae-install gie-install gle-install client coordinator
 	echo "export GRAPHSCOPE_HOME=$(INSTALL_PREFIX)"
 
 clean:
-	rm -rf $(GAE_BUILD_DIR) $(GAE_DIR)/proto
-	cd $(GAE_DIR)/java && mvn clean
+	rm -rf $(ANALYTICAL_BUILD_DIR) $(ANALYTICAL_DIR)/proto
+	cd $(ANALYTICAL_DIR)/java && mvn clean
 
-	cd $(GIE_DIR) && mvn clean
+	cd $(INTERACTIVE_DIR) && mvn clean
     # TODO: use maven clean to clean ir target
-	rm -rf $(GIE_DIR)/executor/ir/target
+	rm -rf $(INTERACTIVE_DIR)/executor/ir/target
 
-	rm -rf $(GLE_BUILD_DIR) $(GLE_DIR)/proto/*.h $(GLE_DIR)/proto/*.cc
+	rm -rf $(LEARNING_BUILD_DIR) $(LEARNING_DIR)/proto/*.h $(LEARNING_DIR)/proto/*.cc
 
 	cd $(CLIENT_DIR) && python3 setup.py clean --all
 
 	cd $(COORDINATOR_DIR) && python3 setup.py clean --all
 
 ## Modules
-.PHONY: client coordinator gae gie gle
+.PHONY: client coordinator analytical interactive learning
+.PHONY: analytical-java
 
-client: gle
+client: learning
 	cd $(CLIENT_DIR) && \
 	pip3 install -r requirements.txt -r requirements-dev.txt --user && \
 	python3 setup.py build_ext --inplace --user
@@ -85,63 +82,81 @@ coordinator: client
 	pip3 install -r requirements.txt -r requirements-dev.txt --user && \
 	python3 setup.py build_builtin
 
-.PHONY: gae-install gie-install gle-install
+# We deliberately make $(ENGINE) depends on a file, and $(ENGINE)-install depends on $(ENGINE),
+# so that when we execute `make $(ENGINE)-install` after `make $(ENGINE)`, it will not
+# rebuild $(ENGINE) from scratch.
+# If we doesn't make gxe depends on a file (as a PHONY), then make will never
+# know if $(ENGINE) is up-to-date or not, so it will always rebuild gxe.
+# Note: `$(ENGINE)` stands for `analytical`, `interactive` and `learning`.
 
-gae-install: gae
-	mkdir -p $(INSTALL_PREFIX)
-	$(MAKE) -C $(GAE_BUILD_DIR) install
-	install $(K8S_DIR)/kube_ssh $(INSTALL_PREFIX)/bin/
+.PHONY: analytical-install interactive-install learning-install
+.PHONY: analytical-java-install
+
+analytical-install: analytical
+	$(MAKE) -C $(ANALYTICAL_BUILD_DIR) install
+	install -d $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake
+	if [ -d "${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical" ]; then \
+		install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/*.cmake $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical; \
+		install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/cmake/* $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake; \
+	fi
+
+analytical: $(ANALYTICAL_BUILD_DIR)/grape_engine
+
+$(ANALYTICAL_BUILD_DIR)/grape_engine:
+	mkdir -p $(ANALYTICAL_BUILD_DIR) && \
+	cd $(ANALYTICAL_BUILD_DIR) && \
+	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
+		-DNETWORKX=$(NETWORKX) \
+		-DBUILD_TESTS=${BUILD_TEST} \
+		-DENABLE_JAVA_SDK=OFF .. && \
+	$(MAKE) -j$(NUMPROC)
+
+analytical-java-install: analytical-java
+	$(MAKE) -C $(ANALYTICAL_BUILD_DIR) install
 	install -d $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake
 	if [ -d "${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical" ]; then \
 		install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/*.cmake $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical; \
 		install $(INSTALL_PREFIX)/lib64/cmake/graphscope-analytical/cmake/* $(INSTALL_PREFIX)/lib/cmake/graphscope-analytical/cmake; \
 	fi
 
-gae: $(GAE_BUILD_DIR)/grape_engine
+analytical-java: $(ANALYTICAL_BUILD_DIR)/graphx_runner
 
-$(GAE_BUILD_DIR)/grape_engine:
-	mkdir -p $(GAE_BUILD_DIR) && \
-	cd $(GAE_BUILD_DIR) && \
+$(ANALYTICAL_BUILD_DIR)/graphx_runner:
+	mkdir -p $(ANALYTICAL_BUILD_DIR) && \
+	cd $(ANALYTICAL_BUILD_DIR) && \
 	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
 		-DNETWORKX=$(NETWORKX) \
 		-DBUILD_TESTS=${BUILD_TEST} \
-		-DENABLE_JAVA_SDK=${ENABLE_JAVA_SDK} .. && \
+		-DENABLE_JAVA_SDK=ON .. && \
 	$(MAKE) -j$(NUMPROC)
 
-gie-install: gie
+interactive-install: interactive
 	mkdir -p $(INSTALL_PREFIX)
-	tar -xf $(GIE_DIR)/assembly/target/graphscope.tar.gz --strip-components 1 -C $(INSTALL_PREFIX)
-gie: $(GIE_DIR)/assembly/target/graphscope.tar.gz
+	tar -xf $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz --strip-components 1 -C $(INSTALL_PREFIX)
+interactive: $(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz
 
-$(GIE_DIR)/assembly/target/graphscope.tar.gz:
-    # frontend/executor
-	cd $(GIE_DIR) && \
+$(INTERACTIVE_DIR)/assembly/target/graphscope.tar.gz:
+	cd $(INTERACTIVE_DIR) && \
 	mvn package -DskipTests -Drust.compile.mode=$(BUILD_TYPE) -P graphscope,graphscope-assembly --quiet
 
-gle-install: gle
+learning-install: learning
 	mkdir -p $(INSTALL_PREFIX)
-	$(MAKE) -C $(GLE_BUILD_DIR) install
-gle: $(GLE_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX)
+	$(MAKE) -C $(LEARNING_BUILD_DIR) install
+learning: $(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX)
 
-$(GLE_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):
+$(LEARNING_DIR)/graphlearn/built/lib/libgraphlearn_shared.$(SUFFIX):
 	git submodule update --init
-	cd $(GLE_DIR) && git submodule update --init third_party/pybind11
-	mkdir -p $(GLE_BUILD_DIR)
-	cd $(GLE_BUILD_DIR) && \
+	cd $(LEARNING_DIR) && git submodule update --init third_party/pybind11
+	mkdir -p $(LEARNING_BUILD_DIR)
+	cd $(LEARNING_BUILD_DIR) && \
 	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) \
 		-DKNN=OFF \
 		-DWITH_VINEYARD=ON \
 		-DTESTING=${BUILD_TEST} .. && \
 	$(MAKE) -j$(NUMPROC)
 
 ## wheels
-.PHONY: graphscope-py3-package graphscope-client-py3-package prepare-client graphscope-docs
-
-graphscope-py3-package:
-	$(MAKE) -C $(K8S_DIR) graphscope-py3-package
-
-graphscope-client-py3-package:
-	 $(MAKE) -C $(K8S_DIR) graphscope-client-py3-package
+.PHONY: prepare-client graphscope-docs
 
 prepare-client:
 	cd $(CLIENT_DIR) && \
@@ -153,25 +168,6 @@ graphscope-docs: prepare-client
 	$(MAKE) -C $(DOCS_DIR)/ html
 
 
-## Images
-.PHONY: graphscope-image jupyter-image dataset-image graphscope-store-image push
-
-graphscope-image:
-	$(MAKE) -C $(K8S_DIR) graphscope-image VERSION=$(VERSION)
-
-jupyter-image:
-	$(MAKE) -C $(K8S_DIR) jupyter-image VERSION=$(VERSION)
-
-dataset-image:
-	$(MAKE) -C $(K8S_DIR) dataset-image VERSION=$(VERSION)
-
-graphscope-store-image:
-	$(MAKE) -C $(K8S_DIR) graphscope-store-image VERSION=$(VERSION)
-
-push:
-	$(MAKE) -C $(K8S_DIR) push
-
-
 ## Tests
 .PHONY: test unittest minitest k8stest
 
@@ -190,3 +186,4 @@ k8stest:
 	pip3 install tensorflow==2.5.2 "pandas<1.5.0"
 	cd $(CLIENT_DIR) && \
 	python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml --cov-report=term -s -v ./graphscope/tests/kubernetes
+