Skip to content
Permalink
Browse files

feat(cli): use ServiceManager as default service runner

  • Loading branch information...
hanxiao committed Aug 8, 2019
1 parent ccfd474 commit 6d3d2b4c4ae216f0a5f3d36ee8310558b1519e82
Showing with 55 additions and 66 deletions.
  1. +1 −1 gnes/__init__.py
  2. +0 −2 gnes/base/__init__.py
  3. +0 −2 gnes/cli/__init__.py
  4. +8 −6 gnes/cli/api.py
  5. +0 −2 gnes/cli/parser.py
  6. +0 −2 gnes/client/http.py
  7. +1 −1 gnes/encoder/__init__.py
  8. +1 −1 gnes/encoder/audio/mfcc.py
  9. +1 −1 gnes/encoder/base.py
  10. +1 −1 gnes/encoder/numeric/hash.py
  11. +1 −1 gnes/encoder/numeric/pca.py
  12. +1 −1 gnes/encoder/numeric/pq.py
  13. +1 −1 gnes/encoder/numeric/tf_pq.py
  14. +1 −1 gnes/encoder/numeric/vlad.py
  15. +1 −1 gnes/encoder/text/bert.py
  16. +1 −1 gnes/encoder/text/elmo.py
  17. +1 −1 gnes/encoder/text/flair.py
  18. +1 −1 gnes/encoder/text/gpt.py
  19. +1 −1 gnes/encoder/text/w2v.py
  20. +1 −1 gnes/helper.py
  21. +1 −1 gnes/indexer/__init__.py
  22. +1 −1 gnes/indexer/base.py
  23. +1 −1 gnes/indexer/fulltext/leveldb.py
  24. +1 −1 gnes/indexer/vector/bindexer/__init__.py
  25. +1 −1 gnes/indexer/vector/faiss.py
  26. +1 −1 gnes/indexer/vector/hbindexer/__init__.py
  27. +1 −1 gnes/indexer/vector/numpy.py
  28. +0 −1 gnes/preprocessor/__init__.py
  29. +1 −1 gnes/preprocessor/base.py
  30. +0 −1 gnes/preprocessor/helper.py
  31. +1 −1 gnes/preprocessor/image/sliding_window.py
  32. +0 −1 gnes/preprocessor/video/shotdetect.py
  33. +0 −1 gnes/router/__init__.py
  34. +1 −1 gnes/router/base.py
  35. +0 −2 gnes/service/__init__.py
  36. +0 −1 gnes/service/base.py
  37. +1 −1 gnes/service/encoder.py
  38. +0 −1 gnes/service/grpc.py
  39. +1 −2 gnes/service/indexer.py
  40. +0 −1 gnes/service/preprocessor.py
  41. +0 −1 gnes/service/router.py
  42. +20 −14 tests/test_service_mgr.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


# do not change this line
# this is managed by git tag and replaced on every release
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import inspect
import os
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import sys

@@ -13,30 +13,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


def preprocess(args):
from ..service.base import ServiceManager
from ..service.preprocessor import PreprocessorService
with PreprocessorService(args) as es:
with ServiceManager(PreprocessorService, args) as es:
es.join()


def encode(args):
from ..service.base import ServiceManager
from ..service.encoder import EncoderService
with EncoderService(args) as es:
with ServiceManager(EncoderService, args) as es:
es.join()


def index(args):
from ..service.base import ServiceManager
from ..service.indexer import IndexerService
with IndexerService(args) as es:
with ServiceManager(IndexerService, args) as es:
es.join()


def route(args):
from ..service.base import ServiceManager
from ..service.router import RouterService
with RouterService(args) as es:
with ServiceManager(RouterService, args) as es:
es.join()


@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import argparse

@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import asyncio
import json
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



# A key-value map for Class to the (module)file it located in
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


from typing import List

@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import List, Any
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import numpy as np
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import numpy as np
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import numpy as np
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import Dict, Any
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import copy
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import List
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import List
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import List
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

# noinspection PyUnresolvedReferences

from typing import List
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



from typing import List
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import fcntl
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


# A key-value map for Class to the (module)file it located in
from ..base import register_all_class
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


from typing import List, Any, Union, Callable, Tuple

@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import pickle
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import os
from typing import List, Tuple, Any
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio



import os
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


import os
from typing import List, Tuple, Any
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio


from typing import List, Tuple, Any

@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

# A key-value map for Class to the (module)file it located in
from ..base import register_all_class
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

import ctypes
import io
import random
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

import datetime
import io
@@ -91,7 +91,7 @@ def _get_all_sliding_window(self, img: 'np.ndarray'):
expanded_input], center_point_list

def _get_slid_offset_nd(self, all_subareas: List[List[int]], index: List[List[int]], center_point: List[float]) -> \
List[int]:
List[int]:
location_list = self._get_location(all_subareas, center_point)
location = [i for i in range(len(location_list)) if location_list[i] is True][0]
return index[location][:2]
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

import numpy as np

@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

# A key-value map for Class to the (module)file it located in
from ..base import register_all_class
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

from typing import List, Generator

from ..base import TrainableBase
@@ -12,5 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio
@@ -14,7 +14,6 @@
# limitations under the License.

import copy
# pylint: disable=low-comment-ratio
import multiprocessing
import random
import threading
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

from typing import List, Union

from .base import BaseService as BS, MessageHandler
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

import threading
import uuid
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

import numpy as np

@@ -70,7 +69,7 @@ def _cal_offset_relevance(q_offset, i_offset):
if not len(q_offset) == 2 and len(i_offset) == 2:
raise ValueError("Length of qc_offset and offset should be (2, 2), "
"but actually we got (%d, %d)" % (len(q_offset), len(i_offset)))
return 1 / (1 + math.sqrt((q_offset[0] - i_offset[0])**2 + (q_offset[1] - i_offset[1])**2))
return 1 / (1 + math.sqrt((q_offset[0] - i_offset[0]) ** 2 + (q_offset[1] - i_offset[1]) ** 2))

vecs = blob2array(msg.request.search.query.chunk_embeddings)
q_offset = [c.offset_nd if msg.request.search.query.doc_type == 'IMAGE'
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

from .base import BaseService as BS, MessageHandler
from ..proto import gnes_pb2
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=low-comment-ratio

from collections import defaultdict
from typing import Dict, List

0 comments on commit 6d3d2b4

Please sign in to comment.
You can’t perform that action at this time.