gss ci use lesser image size (#2186)

* gss ci use lesser image size

* update

.github/workflows/docker/graphscope-store.Dockerfile

@@ -0,0 +1,19 @@
+FROM centos:7.9.2009
+
+RUN yum install -y sudo java-1.8.0-openjdk-devel bind-utils \
+    && yum clean all \
+    && rm -rf /var/cache/yum
+
+COPY k8s/ready_probe.sh /tmp/ready_probe.sh
+ADD artifacts/groot.tar.gz /usr/local
+
+RUN useradd -m graphscope -u 1001 \
+    && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER graphscope
+WORKDIR /home/graphscope
+
+# init log directory
+RUN sudo mkdir /var/log/graphscope \
+  && sudo chown -R $(id -u):$(id -g) /var/log/graphscope
+
+ENV GRAPHSCOPE_HOME=/usr/local

.github/workflows/gss.yml

@@ -114,19 +114,10 @@ jobs:
         retention-days: 5
 
   helm-test:
-    runs-on: [self-hosted, "47.242.9.60"]
+    runs-on: [self-hosted, ubuntu2004]
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     needs: [gremlin-test]
     steps:
-    - name: Clean Up
-      run: |
-        echo "CI is running on host $(curl -s 'https://api.ipify.org')"
-        sudo chmod -R a+wrx ${GITHUB_WORKSPACE}
-        sudo rm -rf ./* || true
-        find ./ -name "*.egg-info" | xargs sudo rm -rf || true
-        find ./ -name "*.whl" | xargs sudo rm -rf || true
-        find ./ -name "*_pb2.py" | xargs sudo rm -rf || true
-        find ./ -name "*_pb2_grpc.py" | xargs sudo rm -rf || true
     - uses: actions/checkout@v3
       with:
         submodules: true
@@ -142,10 +133,18 @@ jobs:
         echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV
 
     - name: Prepare Image
-      env:
-        CI: true
       run: |
-        make graphscope-store-image
+        docker build -t ${{ env.GSS_IMAGE }}:${SHORT_SHA} \
+                -f .github/workflows/docker/graphscope-store.Dockerfile .
+
+    - name: Setup SSH
+      run: |
+        ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ''
+        cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+        chmod og-wx ~/.ssh/authorized_keys
+        echo "StrictHostKeyChecking no" >> ~/.ssh/config
+        sudo /etc/init.d/ssh start
+
     - name: Python Test with Helm Deployment
       env:
         JAVA_HOME: /usr/lib/jvm/default-java
@@ -157,32 +156,38 @@ jobs:
         tar -zxf ./groot.tar.gz
         # download gstest
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
-        # retag image for helm test
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \
-            ${{ env.GSS_IMAGE }}:${SHORT_SHA}
+
+        minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \
+                        --cpus='12' --memory='32000mb' --disk-size='40000mb'
+
+        minikube image load ${{ env.GSS_IMAGE }}:${SHORT_SHA}
+
         # update helm dependency
         cd ${GITHUB_WORKSPACE}/charts/graphscope-store
         helm dependency update
+
         # helm deployment and testing
         cd ${GITHUB_WORKSPACE}/charts
         helm install ci --set image.tag=${SHORT_SHA} ./graphscope-store
         helm test ci --timeout 5m0s
+
         # 1. get gss service endpoint
         export GRPC_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services ci-graphscope-store-frontend)
         export GREMLIN_PORT=$(kubectl get --namespace default -o jsonpath="{.spec.ports[1].nodePort}" services ci-graphscope-store-frontend)
         export NODE_IP=$(kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}")
+
         # 2. deploy hadoop hdfs
-        cd /tmp
-        wget -q https://mirror.cogentco.com/pub/apache/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz
-        tar -zxf hadoop-2.10.1.tar.gz
+        tar -zxf /home/runner/hadoop-2.10.1.tar.gz -C /tmp/
         cd ${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts
+        ./prepare_hadoop.sh /tmp/hadoop-2.10.1
+        export PATH=${PATH}:/tmp/hadoop-2.10.1/bin
+
         REPLACE_STR=${GITHUB_WORKSPACE}/artifacts/groot
         sed s/GROOT_HOME/${REPLACE_STR//\//\\/}/ offline_load.sh.template > offline_load.sh
         chmod +x offline_load.sh
         export LOAD_DATA_SCRIPT=${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts/offline_load.sh
         sed s/GRAPH_ENDPOINT/$NODE_IP:$GRPC_PORT/ databuild.config.template > databuild.config
-        ./prepare_hadoop.sh /tmp/hadoop-2.10.1
-        export PATH=${PATH}:/tmp/hadoop-2.10.1/bin
+
         # 3. upload data to HDFS
         hadoop fs -mkdir /ldbc_sample || true
         hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv
@@ -205,14 +210,3 @@ jobs:
         helm test ci --timeout 5m0s
         cd ${GITHUB_WORKSPACE}/python
         python3 -m pytest -s -vvv graphscope/tests/kubernetes/test_store_service.py -k test_demo_after_restart
-
-    - name: Clean
-      if: always()
-      run: |
-        HADOOP_SSH_OPTS="-o StrictHostKeyChecking=no" /tmp/hadoop-2.10.1/sbin/stop-dfs.sh || true
-        rm -rf /tmp/hadoop* || true
-        helm uninstall ci || true
-        kubectl delete pvc -l app.kubernetes.io/instance=ci --wait=false || true
-        kubectl delete pod ci-graphscope-store-frontend-test-grpc-service ci-graphscope-store-frontend-test-gremlin-service --wait=false || true
-        sudo docker rmi -f graphscope/graphscope-store:${SHORT_SHA}
-        sudo docker rmi -f ${{ env.GSS_IMAGE }}:${SHORT_SHA} || true

k8s/Makefile

@@ -82,7 +82,6 @@ graphscope-store-image:
 	docker pull ${REGISTRY}/$(DOCKER_ORG)/graphscope-vineyard:${VINEYARD_VERSION}
 	docker build --progress=$(BUILD_PROGRESS) \
                  --build-arg profile=${PROFILE} \
-				 --build-arg CI=${CI} \
                  -t graphscope/graphscope-store:${SHORT_SHA} \
                  --network=host \
                  -f $(WORKING_DIR)/graphscope-store.Dockerfile ..

k8s/graphscope-store.Dockerfile

@@ -14,34 +14,28 @@ USER graphscope
 
 RUN sudo chown -R $(id -u):$(id -g) /home/graphscope/gs /home/graphscope/.m2 && \
     cd /home/graphscope/gs && \
-    if [ "${CI}" == "true" ]; then \
-        mv artifacts/groot.tar.gz ./groot.tar.gz; \
-    else \
-        echo "install cppkafka" \
-        && sudo yum update -y && sudo yum install -y librdkafka-devel \
-        && git clone -b 0.4.0 --single-branch --depth=1 https://github.com/mfontanini/cppkafka.git /tmp/cppkafka \
-        && cd /tmp/cppkafka && git submodule update --init \
-        && cmake . && make -j && sudo make install \
-        && echo "build with profile: $profile" \
-        && source ~/.cargo/env \
-        && cd /home/graphscope/gs/interactive_engine \
-        && mvn clean package -P groot,groot-assembly -DskipTests --quiet -Drust.compile.mode="$profile" \
-        && mv /home/graphscope/gs/interactive_engine/assembly/target/groot.tar.gz /home/graphscope/gs/groot.tar.gz; \
-    fi
+    echo "install cppkafka" \
+    && sudo yum update -y && sudo yum install -y librdkafka-devel \
+    && git clone -b 0.4.0 --single-branch --depth=1 https://github.com/mfontanini/cppkafka.git /tmp/cppkafka \
+    && cd /tmp/cppkafka && git submodule update --init \
+    && cmake . && make -j && sudo make install \
+    && echo "build with profile: $profile" \
+    && source ~/.cargo/env \
+    && cd /home/graphscope/gs/interactive_engine \
+    && mvn clean package -P groot,groot-assembly -DskipTests --quiet -Drust.compile.mode="$profile" \
+    && mv /home/graphscope/gs/interactive_engine/assembly/target/groot.tar.gz /home/graphscope/gs/groot.tar.gz; \
 
 FROM centos:7.9.2009
 
-COPY --from=builder /home/graphscope/gs/groot.tar.gz /tmp/groot.tar.gz
-
-COPY ./k8s/ready_probe.sh /tmp/ready_probe.sh
-
-RUN tar -zxf /tmp/groot.tar.gz -C /usr/local
-RUN rm /tmp/groot.tar.gz
-
 RUN yum install -y sudo java-1.8.0-openjdk-devel bind-utils \
     && yum clean all \
     && rm -rf /var/cache/yum
 
+COPY ./k8s/ready_probe.sh /tmp/ready_probe.sh
+
+COPY --from=builder /home/graphscope/gs/groot.tar.gz /tmp/groot.tar.gz
+RUN tar -zxf /tmp/groot.tar.gz -C /usr/local && rm /tmp/groot.tar.gz
+
 RUN useradd -m graphscope -u 1001 \
     && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER graphscope