fix: set docarray version on load flow data (#159)

* fix: set docarray version on load flow data

* test: fix unit tests

* fix: set jcloud versions

* ci: fix python version

* test: update jina new test

* fix: use venv in jina new test

.github/workflows/integration-tests.yml

@@ -33,7 +33,7 @@ jobs:
       max-parallel: 5
       fail-fast: false
       matrix:
-        python-version: [3.7]
+        python-version: ['3.10']
         test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}}
     steps:
       - uses: actions/checkout@v2
@@ -48,6 +48,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install wheel
           pip install --no-cache-dir ".[test]"
+          pip install docarray==0.21.0
           sudo apt-get install libsndfile1
       - name: Test
         id: test

jcloud/helper.py

@@ -387,6 +387,7 @@ def load_flow_data(path: Union[str, Path], envs: Optional[Dict] = None) -> Dict:
         flow_dict = JAML.load(f, substitute=True, context=envs)
         if 'jtype' not in flow_dict or flow_dict['jtype'] != 'Flow':
             raise ValueError(f'The file `{path}` is not a valid Flow YAML')
+        flow_dict = check_and_set_jcloud_versions(flow_dict)
         flow_dict = stringify_labels(flow_dict)
         return flow_dict
 
@@ -434,3 +435,59 @@ def exit_if_flow_defines_secret(flow_path):
             '\n\t2.Create a Secret for your flow with `jc create secret <secret-name> -f <flow-id> --from-literal <secret-data> --update`.',
             'cyan',
         )
+
+
+class JCloudLabelsError(TypeError):
+    pass
+
+
+def stringify(v: Any) -> str:
+    if isinstance(v, str):
+        return v
+    elif isinstance(v, int) or isinstance(v, float):
+        return str(v)
+    else:
+        raise JCloudLabelsError(f'labels can\'t be of type {type(v)}')
+
+
+def stringify_labels(flow_dict: Dict) -> Dict:
+    global_jcloud_labels = flow_dict.get('jcloud', {}).get('labels', None)
+    if global_jcloud_labels:
+        for k, v in flow_dict['jcloud']['labels'].items():
+            flow_dict['jcloud']['labels'][k] = stringify(v)
+    gateway_jcloud_labels = (
+        flow_dict.get('gateway', {}).get('jcloud', {}).get('labels', None)
+    )
+    if gateway_jcloud_labels:
+        for k, v in flow_dict['gateway']['jcloud']['labels'].items():
+            flow_dict['gateway']['jcloud']['labels'][k] = stringify(v)
+
+    executors = flow_dict.get('executors', [])
+    for idx in range(len(executors)):
+        executor_jcloud_labels = (
+            flow_dict['executors'][idx].get('jcloud', {}).get('labels', None)
+        )
+        if executor_jcloud_labels:
+            for k, v in flow_dict['executors'][idx]['jcloud']['labels'].items():
+                flow_dict['executors'][idx]['jcloud']['labels'][k] = stringify(v)
+    return flow_dict
+
+
+def check_and_set_jcloud_versions(flow_dict: Dict) -> Dict:
+    import docarray
+    import jina
+
+    global_jcloud = flow_dict.get('jcloud', None)
+    if not global_jcloud:
+        flow_dict['jcloud'] = {
+            'docarray': docarray.__version__,
+            'version': jina.__version__,
+        }
+        return flow_dict
+    docarray_version = global_jcloud.get('docarray', None)
+    if not docarray_version:
+        flow_dict['jcloud'].update({'docarray': docarray.__version__})
+    jina_version = global_jcloud.get('version', None)
+    if not jina_version:
+        flow_dict['jcloud'].update({'version': jina.__version__})
+    return flow_dict

jcloud/normalize.py

@@ -12,7 +12,11 @@
 import requests
 
 from .constants import CONSTANTS
-from .helper import get_logger, get_filename_envs, load_flow_data
+from .helper import (
+    get_logger,
+    get_filename_envs,
+    load_flow_data,
+)
 
 GPU_DOCKERFILE = 'Dockerfile.gpu'
 

tests/integration/test_jina_new.py

@@ -1,31 +1,47 @@
 import os
+import sys
 import shutil
 import subprocess
+import tempfile
 
 import pytest
-from jina import Client, DocumentArray
 
+from pathlib import Path
+from venv import create
+from jina import Client, DocumentArray
 from jcloud.flow import CloudFlow
 
 cur_dir = os.path.dirname(os.path.abspath(__file__))
 
 
-def test_jina_new():
-    subprocess.run(["jina", "new", os.path.join(cur_dir, "hello-world")])
+def setup_venv():
+    v_dir = Path(tempfile.mkdtemp())
+    create(v_dir, with_pip=True)
+
+    _pip_path = v_dir / 'bin' / 'pip'
+    subprocess.run([_pip_path, 'install', '-U', 'pip', '-q'])
+    subprocess.run([_pip_path, 'install', 'jina[standard]==3.18.0', '-q'])
+    return v_dir
+
 
-    assert os.path.exists(os.path.join(cur_dir, "hello-world"))
-    assert os.path.isdir(os.path.join(cur_dir, "hello-world"))
+def test_jina_new_project():
+    v_dir = setup_venv()
+    subprocess.run(
+        [v_dir / 'bin' / 'jina', 'new', os.path.join(cur_dir, 'hello-world')],
+    )
+    assert os.path.exists(os.path.join(cur_dir, 'hello-world'))
+    assert os.path.isdir(os.path.join(cur_dir, 'hello-world'))
 
-    with CloudFlow(path=os.path.join(cur_dir, "hello-world")) as flow:
+    with CloudFlow(path=os.path.join(cur_dir, 'hello-world')) as flow:
         assert flow.endpoints != {}
         assert 'gateway (grpc)' in flow.endpoints
         assert 'gateway (http)' in flow.endpoints
         assert 'gateway (websocket)' in flow.endpoints
         gateway = flow.endpoints['gateway (grpc)']
 
-        da = Client(host=gateway).post(on="/", inputs=DocumentArray.empty(2))
-        assert da.texts == ["hello, world!", "goodbye, world!"]
+        da = Client(host=gateway).post(on='/', inputs=DocumentArray.empty(2))
+        assert da.texts == ['hello, world!', 'goodbye, world!']
 
-    shutil.rmtree(os.path.join(cur_dir, "hello-world"))
+    shutil.rmtree(os.path.join(cur_dir, 'hello-world'))
 
-    assert not os.path.exists(os.path.join(cur_dir, "hello-world"))
+    assert not os.path.exists(os.path.join(cur_dir, 'hello-world'))

tests/unit/test_helper.py

@@ -10,6 +10,7 @@
     load_flow_data,
     JCloudLabelsError,
     update_flow_yml_and_write_to_file,
+    check_and_set_jcloud_versions,
 )
 from jcloud.env_helper import EnvironmentVariables
 
@@ -76,6 +77,33 @@ def test_not_normalized(filename, envs):
         assert not normalized(f.name)
 
 
+@pytest.mark.parametrize(
+    'flow, flow_dict',
+    (
+        ('flow-one', {'jtype': 'Flow', 'executors': [{'uses': 'jinahub+docker://E1'}]}),
+        (
+            'flow-two',
+            {
+                'jtype': 'Flow',
+                'jcloud': {'docarray': '0.31.0'},
+                'executors': [{'uses': 'jinahub+docker://E1'}],
+            },
+        ),
+    ),
+)
+def test_check_and_set_jcloud_versions(flow, flow_dict):
+    import docarray
+    import jina
+
+    flow_dict = check_and_set_jcloud_versions(flow_dict)
+    if flow == 'flow-one':
+        assert flow_dict['jcloud']['docarray'] == docarray.__version__
+        assert flow_dict['jcloud']['version'] == jina.__version__
+    else:
+        assert flow_dict['jcloud']['docarray'] == '0.31.0'
+        assert flow_dict['jcloud']['version'] == jina.__version__
+
+
 def test_failed_flow():
     flow_path = Path(cur_dir) / 'flows' / 'failed_flows' / 'failed_flow.yml'
 

tests/unit/test_normalize.py

@@ -5,6 +5,7 @@
 from unittest.mock import patch
 from jcloud.helper import load_flow_data
 from jcloud.normalize import *
+from jcloud.helper import JCloudLabelsError
 
 
 @pytest.fixture