Skip to content
Permalink
Browse files

fix(setup): fix setup script

  • Loading branch information...
hanxiao committed Jul 21, 2019
1 parent 26a7ad1 commit 1bef3971dcbe8b35a7ddcfcbdd29011a7f2dc7c0
Showing with 66 additions and 37 deletions.
  1. +28 −2 README.md
  2. +13 −8 gnes/cli/parser.py
  3. +1 −1 gnes/resources/compose/gnes-board.html
  4. +24 −26 setup.py
@@ -60,12 +60,38 @@ GNES enables large-scale index and semantic search for text-to-text, image-to-im
<img src=".github/gnes-component-overview.svg" alt="component overview">
</p>

<h2 align="center">Install</h2>
<h2 align="center">Install GNES</h2>

There are two ways to get GNES, either as a docker image or as a PyPi package.
For cloud users, we highly recommend using GNES as a docker image.

## Run GNES as a Docker Image

#### via [Docker cloud](https://cloud.docker.com/u/gnes/repository/list)

<h2 align="center">Getting Started</h2>
```bash
docker pull gnes/gnes:latest
docker run gnes/gnes:latest --help
```

#### via Tencent Container service

We also provide a public mirror hosted on Tencent Cloud, from which Chinese mainland users can pull the image faster.

```bash
docker login --username=xxx ccr.ccs.tencentyun.com # login to Tencent Cloud so that we can pull from it
docker pull ccr.ccs.tencentyun.com/gnes/gnes:latest
docker run ccr.ccs.tencentyun.com/gnes/gnes:latest --help
```

> You may pull and run different versions by changing the `latest` to a version tag, e.g. `v0.0.24`.
## Install via `pip`

Install


<h2 align="center">Quick Start</h2>

As a cloud-native application, GNES requires an **orchestration engine** to coordinate all micro-services. Currently, we support Kubernetes, Docker Swarm and a built-in solution. Click on one of the icons below to get started.

@@ -18,15 +18,18 @@

import argparse

IDX_PORT_DELTA = 2


def set_base_parser():
from .. import __version__
from termcolor import colored
# create the top-level parser
parser = argparse.ArgumentParser(
description='GNES v%s: Generic Neural Elastic Search '
'is an end-to-end solution for semantic text search' % __version__)
description='%s, a cloud-native semantic search system '
'based on deep neural network. '
'It enables large-scale index and semantic search for text-to-text, image-to-image, '
'video-to-video and any content form. Visit %s for tutorials and documentations.' % (
colored('GNES v%s: Generic Neural Elastic Search' % __version__, 'green'),
colored('https://gnes.ai', 'cyan', attrs=['underline'])))
parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
parser.add_argument('--verbose', action='store_true', default=False,
help='turn on detailed logging for debug')
@@ -49,7 +52,7 @@ def set_composer_parser(parser=None):
parser.add_argument('--yaml_path', type=argparse.FileType('r'),
default=resource_stream(
'gnes', '/'.join(('resources', 'config', 'compose', 'default.yml'))),
help='yaml config of the service')
help='yaml config of the service')
parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'),
default='./gnes-board.html',
help='output path of the HTML file, will contain all possible generations')
@@ -182,7 +185,7 @@ def set_indexer_service_parser(parser=None):

# encoder's port_out is indexer's port_in
parser.set_defaults(port_in=parser.get_default('port_out'),
port_out=parser.get_default('port_out') + IDX_PORT_DELTA,
port_out=parser.get_default('port_out') + 2,
socket_in=SocketType.PULL_CONNECT,
socket_out=SocketType.PUB_BIND)
return parser
@@ -260,7 +263,9 @@ def set_http_service_parser(parser=None):
def get_main_parser():
# create the top-level parser
parser = set_base_parser()
sp = parser.add_subparsers(dest='cli')
sp = parser.add_subparsers(dest='cli', title='GNES sub-commands',
description='use "gnes [sub-command] --help" '
'to get detailed information about each sub-command')

set_grpc_frontend_parser(sp.add_parser('frontend', help='start a grpc frontend service'))
set_indexer_service_parser(sp.add_parser('index', help='start an indexer service'))
@@ -269,5 +274,5 @@ def get_main_parser():
set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
set_http_service_parser(sp.add_parser('client_http', help='start a http service'))
set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
set_composer_flask_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
set_composer_flask_parser(sp.add_parser('compose', help='start a GNES Board and visualize YAML config'))
return parser
@@ -141,7 +141,7 @@
</head>
<body>
<nav class="navbar navbar-light bg-light">
<a class="navbar-brand" href="#">
<a class="navbar-brand" href="https://gnes.ai">
<svg version="1.1" style="height: 30px" viewBox="0.0 0.0 432.0524934383202 157.11023622047244" fill="none"
stroke="none" stroke-linecap="square" stroke-miterlimit="10"
xmlns="http://www.w3.org/2000/svg">
@@ -40,19 +40,29 @@
'ruamel.yaml>=0.15.89',
'pyzmq>=17.1.0',
]
bert_dep = ['bert-serving-server>=1.8.6', 'bert-serving-client>=1.8.6']
elmo_dep = ['elmoformanylangs @ git+https://github.com/HIT-SCIR/ELMoForManyLangs.git@master#egg=elmoformanylangs-0.0.2',
'paramiko', 'pattern3']
flair_dep = ['flair>=0.4.1']
nlp_dep = list(set(bert_dep + flair_dep))
annoy_dep = ['annoy==1.15.2']
chinese_dep = ['jieba']
cn_nlp_dep = list(set(chinese_dep + nlp_dep))
vision_dep = ['opencv-python>=4.0.0', 'torchvision==0.3.0', 'imagehash>=4.0']
leveldb_dep = ['plyvel>=1.0.5']
test_dep = ['pylint', 'memory_profiler>=0.55.0', 'psutil>=5.6.1', 'gputil>=1.4.0']
http_dep = ['flask', 'flask-compress', 'flask-cors', 'flask-json', 'aiohttp==3.5.4']
all_dep = list(set(base_dep + cn_nlp_dep + vision_dep + leveldb_dep + test_dep + annoy_dep))

extras_dep = {
'bert': ['bert-serving-server>=1.8.6', 'bert-serving-client>=1.8.6'],
# 'elmo': [
# 'elmoformanylangs @ git+https://github.com/HIT-SCIR/ELMoForManyLangs.git@master#egg=elmoformanylangs-0.0.2',
# 'paramiko', 'pattern3'],
'flair': ['flair>=0.4.1'],
'annoy': ['annoy==1.15.2'],
'chinese': ['jieba'],
'vision': ['opencv-python>=4.0.0', 'torchvision==0.3.0', 'imagehash>=4.0'],
'leveldb': ['plyvel>=1.0.5'],
'test': ['pylint', 'memory_profiler>=0.55.0', 'psutil>=5.6.1', 'gputil>=1.4.0'],
'http': ['flask', 'flask-compress', 'flask-cors', 'flask-json', 'aiohttp==3.5.4']
}


def combine_dep(new_key, base_keys):
extras_dep[new_key] = list(set(k for v in base_keys for k in extras_dep[v]))


combine_dep('nlp', ['bert', 'flair'])
combine_dep('cn_nlp', ['chinese', 'nlp'])
combine_dep('all', [k for k in extras_dep if k != 'elmo'])

setup(
name=pkg_name,
@@ -75,19 +85,7 @@
],
ext_modules=extensions,
install_requires=base_dep,
extras_require={
'bert': bert_dep,
# 'elmo': elmo_dep, # not welcome by pip
'flair': flair_dep,
'nlp': nlp_dep,
'annoy': annoy_dep,
'chinese': chinese_dep,
'cn_nlp': cn_nlp_dep,
'vision': vision_dep,
'leveldb': leveldb_dep,
'test': test_dep,
'all': all_dep,
},
extras_require=extras_dep,
entry_points={
'console_scripts': ['gnes=gnes.cli:main'],
},

0 comments on commit 1bef397

Please sign in to comment.
You can’t perform that action at this time.