-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
test(daemon): add distribute tests (#1727)
* test(daemon): add distribute tests
- Loading branch information
Showing
7 changed files
with
163 additions
and
23 deletions.
There are no files selected for viewing
Empty file.
17 changes: 17 additions & 0 deletions
17
tests/distributed/test_against_external_daemon/mwu_encoder.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| from typing import Any | ||
|
|
||
| import numpy as np | ||
|
|
||
| from jina.executors.encoders import BaseEncoder | ||
|
|
||
|
|
||
| class MWUEncoder(BaseEncoder): | ||
|
|
||
| def __init__(self, greetings: str, *args, **kwargs): | ||
| super().__init__(*args, **kwargs) | ||
| self._greetings = greetings | ||
| self.logger.success(f'look at me! {greetings}') | ||
|
|
||
| def encode(self, data: Any, *args, **kwargs) -> Any: | ||
| self.logger.info(f'{self._greetings} {data}') | ||
| return np.random.random([data.shape[0], 3]) |
7 changes: 7 additions & 0 deletions
7
tests/distributed/test_against_external_daemon/mwu_encoder.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| !MWUEncoder | ||
| with: | ||
| greetings: im from internal yaml! | ||
| metas: | ||
| name: my-mwu-encoder | ||
| py_modules: mwu_encoder.py | ||
| workspace: ./ |
116 changes: 116 additions & 0 deletions
116
tests/distributed/test_against_external_daemon/test_single_instance.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| import os | ||
|
|
||
| import numpy as np | ||
| import pytest | ||
|
|
||
| from jina import Flow | ||
|
|
||
| CLOUD_HOST = 'cloud.jina.ai:8000' # consider it as the staged version | ||
| NUM_DOCS = 100 | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_r_l_simple(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
| f = (Flow(expose_public=True, parallel=parallels) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| .add(parallel=parallels)) | ||
| with f: | ||
| f.index(('hello' for _ in range(NUM_DOCS)), on_done=response_mock) | ||
|
|
||
| response_mock.assert_called() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_l_r_simple(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
|
|
||
| f = (Flow(expose_public=True, parallel=parallels) | ||
| .add(parallel=parallels) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| ) | ||
| with f: | ||
| f.index(('hello' for _ in range(NUM_DOCS)), on_done=response_mock) | ||
| response_mock.assert_called() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_r_l_r_simple(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
|
|
||
| f = (Flow(expose_public=True) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| .add() | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| ) | ||
| with f: | ||
| f.index(('hello' for _ in range(NUM_DOCS)), on_done=response_mock) | ||
| response_mock.assert_called() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_r_r_r_simple(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
|
|
||
| f = (Flow(expose_public=True) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| ) | ||
| with f: | ||
| f.index(('hello' for _ in range(NUM_DOCS)), on_done=response_mock) | ||
| response_mock.assert_called() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_l_r_l_simple(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
|
|
||
| f = (Flow(expose_public=True) | ||
| .add() | ||
| .add(host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| silent_remote_logs=silent_log) | ||
| .add() | ||
| ) | ||
| with f: | ||
| f.index(('hello' for _ in range(NUM_DOCS)), on_done=response_mock) | ||
| response_mock.assert_called() | ||
|
|
||
|
|
||
| @pytest.mark.skipif('GITHUB_WORKFLOW' in os.environ, | ||
| reason='somehow this upload test does not work on Github action, but locally it works fine!') | ||
| @pytest.mark.parametrize('silent_log', [True, False]) | ||
| @pytest.mark.parametrize('parallels', [1, 2]) | ||
| def test_l_r_l_with_upload(silent_log, parallels, mocker): | ||
| response_mock = mocker.Mock() | ||
| f = (Flow() | ||
| .add() | ||
| .add(uses='mwu_encoder.yml', | ||
| host=CLOUD_HOST, | ||
| parallel=parallels, | ||
| upload_files=['mwu_encoder.py'], | ||
| silent_remote_logs=silent_log) | ||
| .add()) | ||
| with f: | ||
| f.index_ndarray(np.random.random([NUM_DOCS, 100]), on_done=response_mock) | ||
| response_mock.assert_called() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters