[HOTFIX] Fix usage of kubernetes image for v0.7.2 (#2446) (#2447)

.dockerignore

@@ -3,6 +3,8 @@
 .git
 .github
 .idea
+.empty
+.readthedocs.yml
 .vscode
 .pytest*
 .dist-coverage
@@ -16,10 +18,13 @@ misc
 .DS_Store
 .flake8
 .gitignore
-.travis.yml
-appveyor.yml
+.codacy.yml
+.codecov.yml
+azure-pipelines.yml
 *.iml
+*.md
 conda-spec.txt
+CODEOWNERS
 CONTRIBUTING.rst
 LICENSE
 MANIFEST.in

.github/workflows/docker-cd.yml

@@ -0,0 +1,31 @@
+name: Mars CD for DockerHub
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push Docker image
+        shell: bash
+        env:
+          DOCKER_ORG: ${{ secrets.DOCKERHUB_USERNAME }}
+        run: |
+          source ./ci/reload-env.sh
+          if [[ "$DOCKER_ORG" == "marsuploader" ]]; then
+            export DOCKER_ORG="marsproject"
+          fi
+          bash bin/kube-image-tool.sh -o "$DOCKER_ORG" -t "$GIT_TAG" build
+          docker push "$DOCKER_ORG/mars:$GIT_TAG"

.github/workflows/cd.yml → .github/workflows/pypi-cd.yml

@@ -1,4 +1,4 @@
-name: Mars CD
+name: Mars CD for PyPI
 
 on:
   push:

azure-pipelines.yml

@@ -115,6 +115,7 @@ jobs:
       displayName: 'Install conda'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         export DEFAULT_VENV=$VIRTUAL_ENV
 
@@ -125,6 +126,7 @@ jobs:
       displayName: 'Install dependencies'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
 
         # stop the build if there are Python syntax errors or undefined names
@@ -139,27 +141,32 @@ jobs:
       displayName: 'Lint with flake8'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         codespell
       displayName: 'Check spellings'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         git fetch origin master
         bash ci/modecheck.sh
       displayName: 'Check file mode changes'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         python ci/importcheck.py
       displayName: 'Check imports'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         python ci/copycheck.py
       displayName: 'Check copyright headers'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         pushd mars/services/web/ui
         npm install
@@ -168,6 +175,7 @@ jobs:
       displayName: 'Check JS with ESLint'
 
     - bash: |
+        set -e
         source ./ci/reload-env.sh
         pushd docs
         pip install -r requirements-doc.txt

bin/kube-image-tool.sh

@@ -3,14 +3,17 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" && pwd -P)"
 ROOT_PATH=$(dirname "$SCRIPT_PATH")
 
 function usage() {
-  echo "Usage: $0 [-r <repo>] [-t <tag>] (build | push)"
+  echo "Usage: $0 [-r <repo>] [-t <tag>] [-o <organization>] (build | push)"
 }
 
 function image_ref() {
   if [[ -z "$TAG" ]]; then
-    TAG="v$(cd mars || exit 1; python -c "import _version; print(_version.__version__)")"
+    TAG="v$(python -c "import mars; print(mars.__version__)")"
   fi
-  local image_name="marsproject/mars:$TAG"
+  if [[ -z "$ORG" ]]; then
+    ORG="marsproject"
+  fi
+  local image_name="$ORG/mars:$TAG"
   if [[ -n "$REPO" ]]; then
     local image_name="$REPO/$image_name"
   fi
@@ -33,14 +36,17 @@ if [[ "$@" = *--help ]] || [[ "$@" = *-h ]] || [[ "$@" = *-\? ]]; then
   exit 0
 fi
 
-while getopts ":r:t:" option; do
+while getopts ":r:t:o:" option; do
   case "$option" in
     r)
       REPO=$OPTARG
       ;;
     t)
       TAG=$OPTARG
       ;;
+    o)
+      ORG=$OPTARG
+      ;;
     *)
       usage
       exit 1

mars/_version.py

@@ -16,7 +16,7 @@
 import os
 from typing import NamedTuple, Optional
 
-version_info = (0, 7, 2)
+version_info = (0, 7, 2, 'post1')
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \

mars/dataframe/base/describe.py

@@ -190,7 +190,7 @@ def execute(cls, ctx, op):
 
             inputs = [ctx[inp.key] for inp in op.inputs]
             xdf = pd if isinstance(inputs[0], (pd.DataFrame, pd.Series, pd.Index)) \
-                        or cudf is None else cudf
+                or cudf is None else cudf
 
             if len(inputs) == 1:
                 df = inputs[0]

mars/dataframe/base/tests/test_base_execution.py

@@ -943,7 +943,7 @@ def test_to_numeric_execition(setup):
 
     r = to_numeric(l)
     np.testing.assert_array_equal(r.execute().fetch(),
-                                   pd.to_numeric(l))
+                                  pd.to_numeric(l))
 
 
 def test_q_cut_execution(setup):
@@ -1804,7 +1804,7 @@ def test_stack_execution(setup):
             expected = raw2.stack(level=level, dropna=dropna)
             assert_method = \
                 pd.testing.assert_series_equal if expected.ndim == 1 \
-                    else pd.testing.assert_frame_equal
+                else pd.testing.assert_frame_equal
             assert_method(result, expected)
 
 

mars/dataframe/base/transform.py

@@ -100,7 +100,7 @@ def tile(cls, op: "TransformOperand"):
                     chunk_size = chunk_size[::-1]
                 in_df = yield from recursive_tile(in_df.rechunk(chunk_size))
         elif isinstance(op.func, str) or \
-                 (isinstance(op.func, list) and any(isinstance(e, str) for e in op.func)):
+                (isinstance(op.func, list) and any(isinstance(e, str) for e in op.func)):
             # builtin cols handles whole columns, thus merge is needed
             if in_df.chunk_shape[0] > 1:
                 in_df = yield from recursive_tile(in_df.rechunk((in_df.shape[axis],)))

mars/dataframe/datasource/tests/test_datasource_execution.py

@@ -473,7 +473,7 @@ def test_read_csv_execution(setup):
         pd.testing.assert_frame_equal(pdf, mdf)
 
         mdf2 = md.read_csv(file_path, compression='gzip', index_col=0,
-                                                           chunk_bytes='1k').execute().fetch()
+                           chunk_bytes='1k').execute().fetch()
         pd.testing.assert_frame_equal(pdf, mdf2)
 
     # test multiple files

mars/dataframe/reduction/tests/test_reduction.py

@@ -117,7 +117,7 @@ def test_dataframe_reduction(func_name, op, func_opts: FunctionOptions):
 
     assert isinstance(reduction_df, Series)
     assert isinstance(reduction_df.index_value._index_value,
-                          (IndexValue.RangeIndex, IndexValue.Int64Index))
+                      (IndexValue.RangeIndex, IndexValue.Int64Index))
     assert reduction_df.shape == (10,)
 
     reduction_df = tile(reduction_df)

mars/deploy/kubernetes/client.py

@@ -24,6 +24,7 @@
 from ...lib.aio import new_isolation, stop_isolation
 from ...services.cluster.api import WebClusterAPI
 from ...session import new_session
+from ...utils import calc_size_by_str
 from ..utils import wait_services_ready
 from .config import NamespaceConfig, RoleConfig, RoleBindingConfig, ServiceConfig, \
     MarsSupervisorsConfig, MarsWorkersConfig
@@ -119,7 +120,7 @@ def _override_dict(d, updates):
 
         self._supervisor_num = supervisor_num
         self._supervisor_cpu = supervisor_cpu
-        self._supervisor_mem = supervisor_mem
+        self._supervisor_mem = calc_size_by_str(supervisor_mem, None)
         self._supervisor_mem_limit_ratio = supervisor_mem_limit_ratio
         self._supervisor_extra_modules = _override_modules(kwargs.pop('supervisor_extra_modules', []))
         self._supervisor_extra_env = _override_envs(kwargs.pop('supervisor_extra_env', None))
@@ -130,7 +131,7 @@ def _override_dict(d, updates):
 
         self._worker_num = worker_num
         self._worker_cpu = worker_cpu
-        self._worker_mem = worker_mem
+        self._worker_mem = calc_size_by_str(worker_mem, None)
         self._worker_mem_limit_ratio = worker_mem_limit_ratio
         self._worker_spill_paths = worker_spill_paths
         self._worker_cache_mem = worker_cache_mem

mars/deploy/kubernetes/config.py

@@ -18,9 +18,10 @@
 import re
 
 from ... import __version__ as mars_version
-from ...utils import parse_readable_size
+from ...utils import parse_readable_size, calc_size_by_str
 
 DEFAULT_IMAGE = 'marsproject/mars:v' + mars_version
+DEFAULT_WORKER_CACHE_MEM = '40%'
 
 
 def _remove_nones(cfg):
@@ -187,10 +188,10 @@ def memory(self):
         return self._memory
 
     def build(self):
-        return {
-            'cpu': f'{int(self._cpu * 1000)}m',
-            'memory': str(int(self._memory)),
-        }
+        return _remove_nones({
+            'cpu': f'{int(self._cpu * 1000)}m' if self._cpu else None,
+            'memory': str(int(self._memory)) if self._memory else None,
+        })
 
 
 class PortConfig:
@@ -245,9 +246,10 @@ class EmptyDirVolumeConfig(VolumeConfig):
     """
     Configuration builder for Kubernetes empty-dir volumes
     """
-    def __init__(self, name, mount_path, use_memory=False):
+    def __init__(self, name, mount_path, use_memory=True, size_limit=None):
         super().__init__(name, mount_path)
         self._medium = 'Memory' if use_memory else None
+        self._size_limit = size_limit
 
     def build(self):
         result = {
@@ -256,6 +258,8 @@ def build(self):
         }
         if self._medium:
             result['emptyDir']['medium'] = self._medium
+        if self._size_limit:
+            result['emptyDir']['sizeLimit'] = str(int(self._size_limit))
         return result
 
 
@@ -536,7 +540,7 @@ def __init__(self, *args, **kwargs):
         spill_volumes = kwargs.pop('spill_volumes', None) or ()
         mount_shm = kwargs.pop('mount_shm', True)
         self._limit_resources = kwargs['limit_resources'] = kwargs.get('limit_resources', True)
-        worker_cache_mem = kwargs.pop('worker_cache_mem', None)
+        worker_cache_mem = kwargs.pop('worker_cache_mem', None) or DEFAULT_WORKER_CACHE_MEM
         min_cache_mem = kwargs.pop('min_cache_mem', None)
         self._readiness_port = kwargs.pop('readiness_port', self.default_readiness_port)
         supervisor_web_port = kwargs.pop('supervisor_web_port', None)
@@ -555,11 +559,17 @@ def __init__(self, *args, **kwargs):
         if self._spill_volumes:
             self.add_env('MARS_SPILL_DIRS', ':'.join(self._spill_volumes))
 
-        if mount_shm:
-            self.add_env('MARS_K8S_REMOUNT_SHM', '1')
-
-        if worker_cache_mem:
+        if self._memory:
+            size_limit = calc_size_by_str(worker_cache_mem, self._memory)
             self.add_env('MARS_CACHE_MEM_SIZE', worker_cache_mem)
+        else:
+            size_limit = None
+
+        if mount_shm and size_limit:
+            self.add_volume(EmptyDirVolumeConfig(
+                'mars-shared', '/dev/shm', size_limit=size_limit
+            ))
+
         if min_cache_mem:
             self.add_env('MARS_MIN_CACHE_MEM_SIZE', min_cache_mem)
         if supervisor_web_port:

mars/deploy/kubernetes/docker/Dockerfile

@@ -5,11 +5,14 @@ COPY . /opt/mars/
 
 RUN apt-get -yq update --allow-releaseinfo-change \
   && apt-get -yq install gcc g++ \
+  && curl -fsSL https://deb.nodesource.com/setup_14.x | sudo -E bash - \
+  && sudo apt-get install -y nodejs \
   && /opt/conda/bin/pip install -e /opt/mars \
-  && apt-get -yq remove gcc g++ \
+  && apt-get -yq remove gcc g++ nodejs \
   && apt-get -yq autoremove \
   && apt-get -yq clean \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  && rm -rf /usr/local/lib/node_modules
 RUN mkdir -p /srv
 WORKDIR /srv
 

mars/deploy/kubernetes/docker/Dockerfile.base

@@ -6,6 +6,7 @@ COPY retry.sh /srv/retry.sh
 RUN /srv/retry.sh 3 /opt/conda/bin/conda install \
     cloudpickle \
     cython \
+    greenlet \
     mkl \
     numba \
     numexpr \
@@ -14,6 +15,7 @@ RUN /srv/retry.sh 3 /opt/conda/bin/conda install \
     psutil \
     scikit-learn \
     scipy \
+    sqlalchemy \
     tornado \
     lz4 \
   && /srv/retry.sh 3 /opt/conda/bin/conda install -c conda-forge \
@@ -23,3 +25,8 @@ RUN /srv/retry.sh 3 /opt/conda/bin/conda install \
     python-kubernetes \
     uvloop \
   && /opt/conda/bin/conda clean --all -f -y
+
+RUN apt-get -yq update --allow-releaseinfo-change \
+  && apt-get -yq install curl sudo procps \
+  && apt-get -yq clean \
+  && rm -rf /var/lib/apt/lists/* \

mars/deploy/kubernetes/docker/entrypoint.sh

@@ -1,14 +1,6 @@
 #!/bin/bash
 set -e
 
-if [[ $MARS_K8S_REMOUNT_SHM ]]; then
-    if [[ -z "$MARS_CACHE_MEM_SIZE" ]]; then
-        phymem=$(free|awk '/^Mem:/{print $2}')
-        MARS_CACHE_MEM_SIZE=$((phymem * 1000 / 2 ))
-    fi
-    sudo mount -o remount,size=$MARS_CACHE_MEM_SIZE /dev/shm
-fi
-
 if [[ "$1" == *"/"* ]]; then
   $@
 else

mars/deploy/kubernetes/tests/Dockerfile.test

@@ -6,13 +6,12 @@ RUN /srv/retry.sh 3 /opt/conda/bin/conda install -c pkgs/main \
   && conda clean --all -f -y
 
 RUN apt-get -yq update --allow-releaseinfo-change
-RUN apt-get -yq install git gcc g++ procps
+RUN apt-get -yq install git gcc g++
 
 COPY docker-logging.conf /srv/logging.conf
 COPY build_ext.sh /srv/build_ext.sh
 COPY entrypoint.sh /srv/entrypoint.sh
 COPY graceful_stop.sh /srv/graceful_stop.sh
-COPY test_kube_config.yml /srv/config.yml
 
 RUN echo "import coverage; coverage.process_startup()" > \
     $(/opt/conda/bin/python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth