Skip to content

Commit

Permalink
Merge branch 'master' of github.com:jina-ai/jina into refactor-reques…
Browse files Browse the repository at this point in the history
…t-size
  • Loading branch information
JoanFM committed Jan 13, 2021
2 parents c27a9e3 + bfef4f0 commit 3b62d19
Show file tree
Hide file tree
Showing 75 changed files with 37 additions and 37 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
dockerfile: Dockerfiles/pip.Dockerfile
buildargs: PIP_TAG
buildoptions: ${{ matrix.build_option }}
tags: "test-pip"
tags: test-pip${{env.GITHUB_RUN_ID}}
- id: set-matrix
run: |
sudo apt-get install jq
Expand All @@ -104,8 +104,8 @@ jobs:
steps:
- run: |
docker login docker.pkg.github.com -u $GITHUB_ACTOR -p $GITHUB_TOKEN
docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip
docker run docker.pkg.github.com/jina-ai/jina/jina:test-pip hello-world
docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip${{env.GITHUB_RUN_ID}}
docker run docker.pkg.github.com/jina-ai/jina/jina:test-pip${{env.GITHUB_RUN_ID}} hello-world
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
Expand All @@ -127,8 +127,8 @@ jobs:
- name: Prepare enviroment
run: |
docker login docker.pkg.github.com -u $GITHUB_ACTOR -p $GITHUB_TOKEN
docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip
docker tag docker.pkg.github.com/jina-ai/jina/jina:test-pip jinaai/jina:test-pip
docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip${{env.GITHUB_RUN_ID}}
docker tag docker.pkg.github.com/jina-ai/jina/jina:test-pip${{env.GITHUB_RUN_ID}} jinaai/jina:test-pip
python -m pip install --upgrade pip
pip install ".[cicd,test,daemon]" --no-cache-dir
jina check
Expand Down
5 changes: 3 additions & 2 deletions jina/executors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ def __init__(awesomeness = 5):
"""
store_args_kwargs = False #: set this to ``True`` to save ``args`` (in a list) and ``kwargs`` (in a map) in YAML config
exec_methods = ['encode', 'add', 'query', 'craft', 'segment', 'score', 'evaluate', 'predict', 'query_by_id', 'delete', 'update']
exec_methods = ['encode', 'add', 'query', 'craft', 'segment', 'score', 'evaluate', 'predict', 'query_by_id',
'delete', 'update']

def __init__(self, *args, **kwargs):
if isinstance(args, tuple) and len(args) > 0:
Expand Down Expand Up @@ -257,7 +258,7 @@ def current_workspace(self) -> str:
:return: if ``separated_workspace`` is set to ``False`` then ``metas.workspace`` is returned,
otherwise the ``metas.pea_workspace`` is returned
"""
work_dir = self.pea_workspace if self.separated_workspace else self.workspace # type: str
work_dir = self.pea_workspace if self.separated_workspace and self.pea_id != -1 else self.workspace # type: str
return work_dir

def get_file_from_workspace(self, name: str) -> str:
Expand Down
2 changes: 1 addition & 1 deletion jina/jaml/parsers/executor/legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class LegacyParser(VersionedYAMLParser):
@staticmethod
def _get_dump_path_from_config(meta_config: Dict):
if 'name' in meta_config:
if meta_config.get('separated_workspace', False) is True:
if meta_config.get('separated_workspace', False) is True and meta_config['pea_id'] != -1:
if 'pea_id' in meta_config and isinstance(meta_config['pea_id'], int):
work_dir = meta_config['pea_workspace']
dump_path = os.path.join(work_dir, f'{meta_config["name"]}.{"bin"}')
Expand Down
4 changes: 2 additions & 2 deletions scripts/get-all-test-paths.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

set -ex

declare -a array1=( "tests/unit/*.py" "tests/integration/*.py" )
declare -a array2=( $(ls -d tests/{unit,integration}/*/ | grep -v '__pycache__' ))
declare -a array1=( "tests/unit/*.py" "tests/integration/*.py" "tests/distributed/*.py" )
declare -a array2=( $(ls -d tests/{unit,integration,distributed}/*/ | grep -v '__pycache__' ))
dest1=( "${array1[@]}" "${array2[@]}" )

declare -a array1=( "tests/jinad/unit/*.py" "tests/jinad/integration/*.py" )
Expand Down
4 changes: 2 additions & 2 deletions scripts/get-test-paths.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

set -ex

declare -a array1=( "${1}/unit/*.py" "${1}/integration/*.py" )
declare -a array2=( $(ls -d ${1}/{unit,integration}/*/ | grep -v '__pycache__' ))
declare -a array1=( "${1}/unit/*.py" "${1}/integration/*.py" "${1}/distributed/*.py")
declare -a array2=( $(ls -d ${1}/{unit,integration,distributed}/*/ | grep -v '__pycache__' ))
dest=( "${array1[@]}" "${array2[@]}" )
printf '%s\n' "${dest[@]}" | jq -R . | jq -cs .
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ def invoke_requests(method: str,
print(f'got an exception while invoking request {e!r}')
return


def get_results(query: str,
url: str = 'http://0.0.0.0:45678/api/search',
method: str = 'post',
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
- "8000:8000"
- "45678:45678"
env_file:
- tests/jinad/integration/test_index_query/.env
- tests/distributed/test_index_query/.env
expose:
- 10000-60000
encoder:
Expand All @@ -18,11 +18,11 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_index_query/.env
- tests/distributed/test_index_query/.env
indexer:
image: test_index_query
expose:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_index_query/.env
- tests/distributed/test_index_query/.env
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
@pytest.mark.parametrize('docker_compose', [compose_yml], indirect=['docker_compose'])
def test_index_query(docker_compose):
flow_id = create_flow(flow_yml, pod_dir)['flow_id']
print(f'Flow created with id {flow_id}')

r = invoke_requests(method='post',
url='http://0.0.0.0:45678/api/index',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
- "8000:8000"
- "45678:45678"
env_file:
- tests/jinad/integration/test_index_query_with_shards/.env
- tests/distributed/test_index_query_with_shards/.env
expose:
- 10000-60000
encoder:
Expand All @@ -18,11 +18,11 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_index_query_with_shards/.env
- tests/distributed/test_index_query_with_shards/.env
indexer:
image: test_index_query_with_shards
expose:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_index_query_with_shards/.env
- tests/distributed/test_index_query_with_shards/.env
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ services:
- "45005:45005"
- "45006:45006"
env_file:
- tests/jinad/integration/test_local_flow_remote_local_remote/.env
- tests/distributed/test_local_flow_remote_local_remote/.env
expose:
- 10000-60000
networks:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ services:
- "45005:45005"
- "45006:45006"
env_file:
- tests/jinad/integration/test_local_flow_remote_local_remote/.env
- tests/distributed/test_local_flow_remote_local_remote/.env
expose:
- 10000-60000
networks:
Expand All @@ -27,7 +27,7 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_local_flow_remote_local_remote/.env
- tests/distributed/test_local_flow_remote_local_remote/.env
networks:
testing_net:
ipv4_address: 172.28.1.2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
- "8000:8000"
- "45678:45678"
env_file:
- tests/jinad/integration/test_simple_distributed/.env
- tests/distributed/test_simple_distributed/.env
expose:
- 10000-60000
encoder:
Expand All @@ -18,11 +18,11 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_distributed/.env
- tests/distributed/test_simple_distributed/.env
indexer:
image: test_simple_distributed
expose:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_distributed/.env
- tests/distributed/test_simple_distributed/.env
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
- "8000:8000"
- "45678:45678"
env_file:
- tests/jinad/integration/test_simple_distributed_with_shards/.env
- tests/distributed/test_simple_distributed_with_shards/.env
expose:
- 10000-60000
pod1:
Expand All @@ -18,11 +18,11 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_distributed_with_shards/.env
- tests/distributed/test_simple_distributed_with_shards/.env
pod2:
image: test_simple_distributed_with_shards
expose:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_distributed_with_shards/.env
- tests/distributed/test_simple_distributed_with_shards/.env
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ services:
- "8000:8000"
- "45678:45678"
env_file:
- tests/jinad/integration/test_simple_hub_pods/.env
- tests/distributed/test_simple_hub_pods/.env
expose:
- 10000-60000
volumes:
Expand All @@ -20,7 +20,7 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_hub_pods/.env
- tests/distributed/test_simple_hub_pods/.env
volumes:
- /var/run/docker.sock:/var/run/docker.sock
pod2:
Expand All @@ -29,6 +29,6 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_hub_pods/.env
- tests/distributed/test_simple_hub_pods/.env
volumes:
- /var/run/docker.sock:/var/run/docker.sock
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ services:
- "45001:45001"
- "45002:45002"
env_file:
- tests/jinad/integration/test_simple_local_remote/.env
- tests/distributed/test_simple_local_remote/.env
expose:
- 10000-60000
networks:
Expand All @@ -23,7 +23,7 @@ services:
- 8000
- 10000-60000
env_file:
- tests/jinad/integration/test_simple_local_remote/.env
- tests/distributed/test_simple_local_remote/.env
networks:
testing_net:
ipv4_address: 172.28.1.2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,14 @@ def test_unique_indexing_vecindexers(random_workspace, restful):
assert vector_indexer.size == num_uniq_docs


@pytest.mark.parametrize('separated_workspace', [False, True])
@pytest.mark.parametrize('restful', [False, True])
def test_unique_indexing_docindexers(random_workspace, restful):
def test_unique_indexing_docindexers(random_workspace, restful, separated_workspace):
total_docs = 10
duplicate_docs, num_uniq_docs = get_duplicate_docs(num_docs=total_docs)

f = (Flow(restful=restful)
.add(uses=os.path.join(cur_dir, 'uniq_docindexer.yml'), shards=1))
.add(uses=os.path.join(cur_dir, 'uniq_docindexer.yml'), shards=1, separated_workspace=separated_workspace))

with f:
f.index(duplicate_docs)
Expand Down
2 changes: 0 additions & 2 deletions tests/unit/test_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ def test_index(test_workspace_index):
f = Flow().add(uses=os.path.join(cur_dir, 'yaml/test-index.yml'), parallel=3, separated_workspace=True)
with f:
f.index(input_fn=random_docs(50))

for j in range(3):
assert os.path.join(test_workspace_index, f'test2-{j + 1}/test2.bin')
assert os.path.exists(os.path.join(test_workspace_index, f'test2-{j + 1}/tmp2'))


Expand Down

0 comments on commit 3b62d19

Please sign in to comment.