From a2ddcae04355b9956ec3d282028fbbb0d920c176 Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Fri, 23 Jun 2023 12:35:10 +0100 Subject: [PATCH 1/6] fix: set docarray version on load flow data --- jcloud/helper.py | 49 ++++++++++++++++++++++++++++++++++++ jcloud/normalize.py | 8 +++++- tests/unit/test_helper.py | 26 +++++++++++++++++++ tests/unit/test_normalize.py | 1 + 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/jcloud/helper.py b/jcloud/helper.py index b3cee20..3b0fe0e 100644 --- a/jcloud/helper.py +++ b/jcloud/helper.py @@ -3,6 +3,7 @@ import logging import os import sys +import requests import threading import warnings from datetime import datetime @@ -434,3 +435,51 @@ def exit_if_flow_defines_secret(flow_path): '\n\t2.Create a Secret for your flow with `jc create secret -f --from-literal --update`.', 'cyan', ) + + +class JCloudLabelsError(TypeError): + pass + + +def stringify(v: Any) -> str: + if isinstance(v, str): + return v + elif isinstance(v, int) or isinstance(v, float): + return str(v) + else: + raise JCloudLabelsError(f'labels can\'t be of type {type(v)}') + + +def stringify_labels(flow_dict: Dict) -> Dict: + global_jcloud_labels = flow_dict.get('jcloud', {}).get('labels', None) + if global_jcloud_labels: + for k, v in flow_dict['jcloud']['labels'].items(): + flow_dict['jcloud']['labels'][k] = stringify(v) + gateway_jcloud_labels = ( + flow_dict.get('gateway', {}).get('jcloud', {}).get('labels', None) + ) + if gateway_jcloud_labels: + for k, v in flow_dict['gateway']['jcloud']['labels'].items(): + flow_dict['gateway']['jcloud']['labels'][k] = stringify(v) + + executors = flow_dict.get('executors', []) + for idx in range(len(executors)): + executor_jcloud_labels = ( + flow_dict['executors'][idx].get('jcloud', {}).get('labels', None) + ) + if executor_jcloud_labels: + for k, v in flow_dict['executors'][idx]['jcloud']['labels'].items(): + flow_dict['executors'][idx]['jcloud']['labels'][k] = stringify(v) + return flow_dict + + +def get_docarray_latest_version_from_pypi(): + resp = requests.get('https://pypi.python.org/pypi/docarray/json') + return resp.json()['info']['version'] + + +def check_and_set_docarray_version(flow_dict: Dict) -> Dict: + docarray_version = flow_dict.get('jcloud', {}).get('docarray', None) + if not docarray_version: + flow_dict['jcloud'] = {'docarray': get_docarray_latest_version_from_pypi()} + return flow_dict diff --git a/jcloud/normalize.py b/jcloud/normalize.py index b6c12b3..b90041e 100644 --- a/jcloud/normalize.py +++ b/jcloud/normalize.py @@ -12,7 +12,13 @@ import requests from .constants import CONSTANTS -from .helper import get_logger, get_filename_envs, load_flow_data +from .helper import ( + get_logger, + stringify_labels, + check_and_set_docarray_version, + get_filename_envs, + load_flow_data, +) GPU_DOCKERFILE = 'Dockerfile.gpu' diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index feb5b18..f5218f1 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -10,6 +10,8 @@ load_flow_data, JCloudLabelsError, update_flow_yml_and_write_to_file, + check_and_set_docarray_version, + get_docarray_latest_version_from_pypi, ) from jcloud.env_helper import EnvironmentVariables @@ -76,6 +78,30 @@ def test_not_normalized(filename, envs): assert not normalized(f.name) +@pytest.mark.parametrize( + 'flow, flow_dict', + ( + ('flow-one', {'jtype': 'Flow', 'executors': [{'uses': 'jinahub+docker://E1'}]}), + ( + 'flow-two', + { + 'jtype': 'Flow', + 'jcloud': {'docarray': '0.31.0'}, + 'executors': [{'uses': 'jinahub+docker://E1'}], + }, + ), + ), +) +def test_check_and_set_docarray_version(flow, flow_dict): + flow_dict = check_and_set_docarray_version(flow_dict) + if flow == 'flow-one': + assert ( + flow_dict['jcloud']['docarray'] == get_docarray_latest_version_from_pypi() + ) + else: + assert flow_dict['jcloud']['docarray'] == '0.31.0' + + def test_failed_flow(): flow_path = Path(cur_dir) / 'flows' / 'failed_flows' / 'failed_flow.yml' diff --git a/tests/unit/test_normalize.py b/tests/unit/test_normalize.py index 4b8ede9..8669be1 100644 --- a/tests/unit/test_normalize.py +++ b/tests/unit/test_normalize.py @@ -5,6 +5,7 @@ from unittest.mock import patch from jcloud.helper import load_flow_data from jcloud.normalize import * +from jcloud.helper import JCloudLabelsError @pytest.fixture From 40503e11fce2acf1834ba2246c22aa2c1f509264 Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Fri, 23 Jun 2023 12:50:01 +0100 Subject: [PATCH 2/6] test: fix unit tests --- jcloud/helper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/jcloud/helper.py b/jcloud/helper.py index 3b0fe0e..e4d8599 100644 --- a/jcloud/helper.py +++ b/jcloud/helper.py @@ -479,7 +479,13 @@ def get_docarray_latest_version_from_pypi(): def check_and_set_docarray_version(flow_dict: Dict) -> Dict: - docarray_version = flow_dict.get('jcloud', {}).get('docarray', None) - if not docarray_version: + global_jcloud = flow_dict.get('jcloud', None) + if not global_jcloud: flow_dict['jcloud'] = {'docarray': get_docarray_latest_version_from_pypi()} + return flow_dict + docarray = global_jcloud.get('docarray', None) + if not docarray: + flow_dict['jcloud'].update( + {'docarray': get_docarray_latest_version_from_pypi()} + ) return flow_dict From c07fae0f61ffdd35c0dfa3c85c536bc1857ad22f Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Tue, 11 Jul 2023 11:02:28 +0100 Subject: [PATCH 3/6] fix: set jcloud versions --- .github/workflows/integration-tests.yml | 3 ++- jcloud/helper.py | 26 +++++++++++++------------ jcloud/normalize.py | 2 -- tests/unit/test_helper.py | 16 ++++++++------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e245316..f6b2238 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,7 +33,7 @@ jobs: max-parallel: 5 fail-fast: false matrix: - python-version: [3.7] + python-version: [3.10] test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} steps: - uses: actions/checkout@v2 @@ -48,6 +48,7 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install --no-cache-dir ".[test]" + pip install docarray==0.21.0 sudo apt-get install libsndfile1 - name: Test id: test diff --git a/jcloud/helper.py b/jcloud/helper.py index e4d8599..2dc4ccb 100644 --- a/jcloud/helper.py +++ b/jcloud/helper.py @@ -3,7 +3,6 @@ import logging import os import sys -import requests import threading import warnings from datetime import datetime @@ -388,6 +387,7 @@ def load_flow_data(path: Union[str, Path], envs: Optional[Dict] = None) -> Dict: flow_dict = JAML.load(f, substitute=True, context=envs) if 'jtype' not in flow_dict or flow_dict['jtype'] != 'Flow': raise ValueError(f'The file `{path}` is not a valid Flow YAML') + flow_dict = check_and_set_jcloud_versions(flow_dict) flow_dict = stringify_labels(flow_dict) return flow_dict @@ -473,19 +473,21 @@ def stringify_labels(flow_dict: Dict) -> Dict: return flow_dict -def get_docarray_latest_version_from_pypi(): - resp = requests.get('https://pypi.python.org/pypi/docarray/json') - return resp.json()['info']['version'] +def check_and_set_jcloud_versions(flow_dict: Dict) -> Dict: + import docarray + import jina - -def check_and_set_docarray_version(flow_dict: Dict) -> Dict: global_jcloud = flow_dict.get('jcloud', None) if not global_jcloud: - flow_dict['jcloud'] = {'docarray': get_docarray_latest_version_from_pypi()} + flow_dict['jcloud'] = { + 'docarray': docarray.__version__, + 'version': jina.__version__, + } return flow_dict - docarray = global_jcloud.get('docarray', None) - if not docarray: - flow_dict['jcloud'].update( - {'docarray': get_docarray_latest_version_from_pypi()} - ) + docarray_version = global_jcloud.get('docarray', None) + if not docarray_version: + flow_dict['jcloud'].update({'docarray': docarray.__version__}) + jina_version = global_jcloud.get('version', None) + if not jina_version: + flow_dict['jcloud'].update({'version': jina.__version__}) return flow_dict diff --git a/jcloud/normalize.py b/jcloud/normalize.py index b90041e..80ca68c 100644 --- a/jcloud/normalize.py +++ b/jcloud/normalize.py @@ -14,8 +14,6 @@ from .constants import CONSTANTS from .helper import ( get_logger, - stringify_labels, - check_and_set_docarray_version, get_filename_envs, load_flow_data, ) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index f5218f1..783d206 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -10,8 +10,7 @@ load_flow_data, JCloudLabelsError, update_flow_yml_and_write_to_file, - check_and_set_docarray_version, - get_docarray_latest_version_from_pypi, + check_and_set_jcloud_versions, ) from jcloud.env_helper import EnvironmentVariables @@ -92,14 +91,17 @@ def test_not_normalized(filename, envs): ), ), ) -def test_check_and_set_docarray_version(flow, flow_dict): - flow_dict = check_and_set_docarray_version(flow_dict) +def test_check_and_set_jcloud_versions(flow, flow_dict): + import docarray + import jina + + flow_dict = check_and_set_jcloud_versions(flow_dict) if flow == 'flow-one': - assert ( - flow_dict['jcloud']['docarray'] == get_docarray_latest_version_from_pypi() - ) + assert flow_dict['jcloud']['docarray'] == docarray.__version__ + assert flow_dict['jcloud']['version'] == jina.__version__ else: assert flow_dict['jcloud']['docarray'] == '0.31.0' + assert flow_dict['jcloud']['version'] == jina.__version__ def test_failed_flow(): From ac81775bdc09e9f90066c9e534fc5c05f6b735c8 Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Tue, 11 Jul 2023 11:07:44 +0100 Subject: [PATCH 4/6] ci: fix python version --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f6b2238..4b076ea 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,7 +33,7 @@ jobs: max-parallel: 5 fail-fast: false matrix: - python-version: [3.10] + python-version: ['3.10'] test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} steps: - uses: actions/checkout@v2 From 4329d7ef51986364a42a1fd32860d09ec90b2ad7 Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Tue, 11 Jul 2023 16:13:17 +0100 Subject: [PATCH 5/6] test: update jina new test --- tests/integration/test_jina_new.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_jina_new.py b/tests/integration/test_jina_new.py index 8ddb0e4..94e1ae2 100644 --- a/tests/integration/test_jina_new.py +++ b/tests/integration/test_jina_new.py @@ -1,20 +1,30 @@ import os +import pip import shutil import subprocess import pytest -from jina import Client, DocumentArray from jcloud.flow import CloudFlow +from jina import Client cur_dir = os.path.dirname(os.path.abspath(__file__)) def test_jina_new(): - subprocess.run(["jina", "new", os.path.join(cur_dir, "hello-world")]) + subprocess.run(['jina', 'new', os.path.join(cur_dir, 'hello-world')]) + import docarray - assert os.path.exists(os.path.join(cur_dir, "hello-world")) - assert os.path.isdir(os.path.join(cur_dir, "hello-world")) + print(docarray.__version__) + subprocess.run(['pip', 'install', '-U', 'docarray', '-q']) + docarray = __import__('docarray') + + print(docarray.__version__) + from docarray import DocList + from docarray.documents import TextDoc + + assert os.path.exists(os.path.join(cur_dir, 'hello-world')) + assert os.path.isdir(os.path.join(cur_dir, 'hello-world')) with CloudFlow(path=os.path.join(cur_dir, "hello-world")) as flow: assert flow.endpoints != {} @@ -23,8 +33,11 @@ def test_jina_new(): assert 'gateway (websocket)' in flow.endpoints gateway = flow.endpoints['gateway (grpc)'] - da = Client(host=gateway).post(on="/", inputs=DocumentArray.empty(2)) - assert da.texts == ["hello, world!", "goodbye, world!"] + da = Client(host=gateway).post( + on='/', inputs=DocList[TextDoc](TextDoc() for i in range(2)) + ) + assert da[0].text == ['hello, world!'] + assert da[1].text == ['goodbye, world!'] shutil.rmtree(os.path.join(cur_dir, "hello-world")) From c3a6e44a7060889815e15a0b5751204f0ba8de04 Mon Sep 17 00:00:00 2001 From: Nikos Pitsillos Date: Wed, 12 Jul 2023 08:50:36 +0100 Subject: [PATCH 6/6] fix: use venv in jina new test --- tests/integration/test_jina_new.py | 41 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_jina_new.py b/tests/integration/test_jina_new.py index 94e1ae2..8ab6888 100644 --- a/tests/integration/test_jina_new.py +++ b/tests/integration/test_jina_new.py @@ -1,44 +1,47 @@ import os -import pip +import sys import shutil import subprocess +import tempfile import pytest +from pathlib import Path +from venv import create +from jina import Client, DocumentArray from jcloud.flow import CloudFlow -from jina import Client cur_dir = os.path.dirname(os.path.abspath(__file__)) -def test_jina_new(): - subprocess.run(['jina', 'new', os.path.join(cur_dir, 'hello-world')]) - import docarray +def setup_venv(): + v_dir = Path(tempfile.mkdtemp()) + create(v_dir, with_pip=True) - print(docarray.__version__) - subprocess.run(['pip', 'install', '-U', 'docarray', '-q']) - docarray = __import__('docarray') + _pip_path = v_dir / 'bin' / 'pip' + subprocess.run([_pip_path, 'install', '-U', 'pip', '-q']) + subprocess.run([_pip_path, 'install', 'jina[standard]==3.18.0', '-q']) + return v_dir - print(docarray.__version__) - from docarray import DocList - from docarray.documents import TextDoc +def test_jina_new_project(): + v_dir = setup_venv() + subprocess.run( + [v_dir / 'bin' / 'jina', 'new', os.path.join(cur_dir, 'hello-world')], + ) assert os.path.exists(os.path.join(cur_dir, 'hello-world')) assert os.path.isdir(os.path.join(cur_dir, 'hello-world')) - with CloudFlow(path=os.path.join(cur_dir, "hello-world")) as flow: + with CloudFlow(path=os.path.join(cur_dir, 'hello-world')) as flow: assert flow.endpoints != {} assert 'gateway (grpc)' in flow.endpoints assert 'gateway (http)' in flow.endpoints assert 'gateway (websocket)' in flow.endpoints gateway = flow.endpoints['gateway (grpc)'] - da = Client(host=gateway).post( - on='/', inputs=DocList[TextDoc](TextDoc() for i in range(2)) - ) - assert da[0].text == ['hello, world!'] - assert da[1].text == ['goodbye, world!'] + da = Client(host=gateway).post(on='/', inputs=DocumentArray.empty(2)) + assert da.texts == ['hello, world!', 'goodbye, world!'] - shutil.rmtree(os.path.join(cur_dir, "hello-world")) + shutil.rmtree(os.path.join(cur_dir, 'hello-world')) - assert not os.path.exists(os.path.join(cur_dir, "hello-world")) + assert not os.path.exists(os.path.join(cur_dir, 'hello-world'))