Skip to content
This repository has been archived by the owner on Feb 22, 2020. It is now read-only.

Commit

Permalink
fix(setup): fix setup script
Browse files Browse the repository at this point in the history
  • Loading branch information
hanhxiao committed Jul 21, 2019
1 parent 26a7ad1 commit 1bef397
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 37 deletions.
30 changes: 28 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,38 @@ GNES enables large-scale index and semantic search for text-to-text, image-to-im
<img src=".github/gnes-component-overview.svg" alt="component overview">
</p>

<h2 align="center">Install</h2>
<h2 align="center">Install GNES</h2>

There are two ways to get GNES, either as a docker image or as a PyPi package.
For cloud users, we highly recommend using GNES as a docker image.

## Run GNES as a Docker Image

#### via [Docker cloud](https://cloud.docker.com/u/gnes/repository/list)

<h2 align="center">Getting Started</h2>
```bash
docker pull gnes/gnes:latest
docker run gnes/gnes:latest --help
```

#### via Tencent Container service

We also provide a public mirror hosted on Tencent Cloud, from which Chinese mainland users can pull the image faster.

```bash
docker login --username=xxx ccr.ccs.tencentyun.com # login to Tencent Cloud so that we can pull from it
docker pull ccr.ccs.tencentyun.com/gnes/gnes:latest
docker run ccr.ccs.tencentyun.com/gnes/gnes:latest --help
```

> You may pull and run different versions by changing the `latest` to a version tag, e.g. `v0.0.24`.
## Install via `pip`

Install


<h2 align="center">Quick Start</h2>

As a cloud-native application, GNES requires an **orchestration engine** to coordinate all micro-services. Currently, we support Kubernetes, Docker Swarm and a built-in solution. Click on one of the icons below to get started.

Expand Down
21 changes: 13 additions & 8 deletions gnes/cli/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@

import argparse

IDX_PORT_DELTA = 2


def set_base_parser():
from .. import __version__
from termcolor import colored
# create the top-level parser
parser = argparse.ArgumentParser(
description='GNES v%s: Generic Neural Elastic Search '
'is an end-to-end solution for semantic text search' % __version__)
description='%s, a cloud-native semantic search system '
'based on deep neural network. '
'It enables large-scale index and semantic search for text-to-text, image-to-image, '
'video-to-video and any content form. Visit %s for tutorials and documentations.' % (
colored('GNES v%s: Generic Neural Elastic Search' % __version__, 'green'),
colored('https://gnes.ai', 'cyan', attrs=['underline'])))
parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
parser.add_argument('--verbose', action='store_true', default=False,
help='turn on detailed logging for debug')
Expand All @@ -49,7 +52,7 @@ def set_composer_parser(parser=None):
parser.add_argument('--yaml_path', type=argparse.FileType('r'),
default=resource_stream(
'gnes', '/'.join(('resources', 'config', 'compose', 'default.yml'))),
help='yaml config of the service')
help='yaml config of the service')
parser.add_argument('--html_path', type=argparse.FileType('w', encoding='utf8'),
default='./gnes-board.html',
help='output path of the HTML file, will contain all possible generations')
Expand Down Expand Up @@ -182,7 +185,7 @@ def set_indexer_service_parser(parser=None):

# encoder's port_out is indexer's port_in
parser.set_defaults(port_in=parser.get_default('port_out'),
port_out=parser.get_default('port_out') + IDX_PORT_DELTA,
port_out=parser.get_default('port_out') + 2,
socket_in=SocketType.PULL_CONNECT,
socket_out=SocketType.PUB_BIND)
return parser
Expand Down Expand Up @@ -260,7 +263,9 @@ def set_http_service_parser(parser=None):
def get_main_parser():
# create the top-level parser
parser = set_base_parser()
sp = parser.add_subparsers(dest='cli')
sp = parser.add_subparsers(dest='cli', title='GNES sub-commands',
description='use "gnes [sub-command] --help" '
'to get detailed information about each sub-command')

set_grpc_frontend_parser(sp.add_parser('frontend', help='start a grpc frontend service'))
set_indexer_service_parser(sp.add_parser('index', help='start an indexer service'))
Expand All @@ -269,5 +274,5 @@ def get_main_parser():
set_preprocessor_service_parser(sp.add_parser('preprocess', help='start a preprocessor service'))
set_http_service_parser(sp.add_parser('client_http', help='start a http service'))
set_cli_client_parser(sp.add_parser('client_cli', help='start a grpc client'))
set_composer_flask_parser(sp.add_parser('compose', help='start a GNES composer to simplify config generation'))
set_composer_flask_parser(sp.add_parser('compose', help='start a GNES Board and visualize YAML config'))
return parser
2 changes: 1 addition & 1 deletion gnes/resources/compose/gnes-board.html
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@
</head>
<body>
<nav class="navbar navbar-light bg-light">
<a class="navbar-brand" href="#">
<a class="navbar-brand" href="https://gnes.ai">
<svg version="1.1" style="height: 30px" viewBox="0.0 0.0 432.0524934383202 157.11023622047244" fill="none"
stroke="none" stroke-linecap="square" stroke-miterlimit="10"
xmlns="http://www.w3.org/2000/svg">
Expand Down
50 changes: 24 additions & 26 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,29 @@
'ruamel.yaml>=0.15.89',
'pyzmq>=17.1.0',
]
bert_dep = ['bert-serving-server>=1.8.6', 'bert-serving-client>=1.8.6']
elmo_dep = ['elmoformanylangs @ git+https://github.com/HIT-SCIR/ELMoForManyLangs.git@master#egg=elmoformanylangs-0.0.2',
'paramiko', 'pattern3']
flair_dep = ['flair>=0.4.1']
nlp_dep = list(set(bert_dep + flair_dep))
annoy_dep = ['annoy==1.15.2']
chinese_dep = ['jieba']
cn_nlp_dep = list(set(chinese_dep + nlp_dep))
vision_dep = ['opencv-python>=4.0.0', 'torchvision==0.3.0', 'imagehash>=4.0']
leveldb_dep = ['plyvel>=1.0.5']
test_dep = ['pylint', 'memory_profiler>=0.55.0', 'psutil>=5.6.1', 'gputil>=1.4.0']
http_dep = ['flask', 'flask-compress', 'flask-cors', 'flask-json', 'aiohttp==3.5.4']
all_dep = list(set(base_dep + cn_nlp_dep + vision_dep + leveldb_dep + test_dep + annoy_dep))

extras_dep = {
'bert': ['bert-serving-server>=1.8.6', 'bert-serving-client>=1.8.6'],
# 'elmo': [
# 'elmoformanylangs @ git+https://github.com/HIT-SCIR/ELMoForManyLangs.git@master#egg=elmoformanylangs-0.0.2',
# 'paramiko', 'pattern3'],
'flair': ['flair>=0.4.1'],
'annoy': ['annoy==1.15.2'],
'chinese': ['jieba'],
'vision': ['opencv-python>=4.0.0', 'torchvision==0.3.0', 'imagehash>=4.0'],
'leveldb': ['plyvel>=1.0.5'],
'test': ['pylint', 'memory_profiler>=0.55.0', 'psutil>=5.6.1', 'gputil>=1.4.0'],
'http': ['flask', 'flask-compress', 'flask-cors', 'flask-json', 'aiohttp==3.5.4']
}


def combine_dep(new_key, base_keys):
extras_dep[new_key] = list(set(k for v in base_keys for k in extras_dep[v]))


combine_dep('nlp', ['bert', 'flair'])
combine_dep('cn_nlp', ['chinese', 'nlp'])
combine_dep('all', [k for k in extras_dep if k != 'elmo'])

setup(
name=pkg_name,
Expand All @@ -75,19 +85,7 @@
],
ext_modules=extensions,
install_requires=base_dep,
extras_require={
'bert': bert_dep,
# 'elmo': elmo_dep, # not welcome by pip
'flair': flair_dep,
'nlp': nlp_dep,
'annoy': annoy_dep,
'chinese': chinese_dep,
'cn_nlp': cn_nlp_dep,
'vision': vision_dep,
'leveldb': leveldb_dep,
'test': test_dep,
'all': all_dep,
},
extras_require=extras_dep,
entry_points={
'console_scripts': ['gnes=gnes.cli:main'],
},
Expand Down

0 comments on commit 1bef397

Please sign in to comment.