[EN-3260] fix build, clear tests

Author: ttldtor

.gitignore

@@ -21,3 +21,7 @@ dist
 build/**
 dxfeed/core/**/*.c
 dxfeed/tmp
+
+setup.py
+poetry.lock
+dxfeed.egg-info

README.md

@@ -34,12 +34,6 @@ pip3 install dxfeed
 
 ## Installation from sources
 
-Reminder: initialize and pull git submodule after cloning the repo:
-```bash
-git submodule init
-git submodule update
-``` 
-
 To install dxfeed from source you need Poetry. It provides a custom installer.
 This is the recommended way of installing poetry according to [documentation](https://python-poetry.org/docs/)
 

build.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import struct
 
 from setuptools import Extension, find_packages
@@ -32,45 +33,66 @@
 else:
     current_os = 'centos'
 
+capi_root_dir = root_path / 'dxfeed' / 'dxfeed-c-api'
+capi_include_dir = capi_root_dir / 'include'
+# capi_bin_dir = capi_root_dir / 'bin'
+capi_bin_dir = root_path / 'dxfeed' / 'core'
 path_to_extract = root_path / 'dxfeed' / 'tmp'
-capi_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls'
+capi_extracted_root_dir = path_to_extract / f'dxfeed-c-api-{capi_version}-no-tls'
 
-if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_root_dir)):
-    url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-{current_os}-no-tls.zip'
+if (not os.path.exists(path_to_extract)) or (not os.path.exists(capi_extracted_root_dir)):
+    url = f'https://github.com/dxFeed/dxfeed-c-api/releases/download/{capi_version}/dxfeed-c-api-{capi_version}-' \
+          f'{current_os}-no-tls.zip '
     print(f'Downloading the "{url}"')
     resp = urlopen(url)
     zipfile = ZipFile(BytesIO(resp.read()))
     print(f'Extracting to "{path_to_extract}"')
     zipfile.extractall(path_to_extract)
 
-if current_os == 'windows':
+if platform.system() == 'Windows':
     if is_x64:
-        capi_library_dir = str(capi_root_dir / 'bin' / 'x64')
+        capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64'
         capi_library_name = 'DXFeed_64'
+        capi_library_file_name = f'{capi_library_name}.dll'
+        capi_library_file_name2 = f'{capi_library_name}.lib'
     else:
-        capi_library_dir = capi_root_dir / 'bin' / 'x86'
+        capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x86'
         capi_library_name = 'DXFeed'
-elif current_os == 'macosx':
+        capi_library_file_name = f'{capi_library_name}.dll'
+        capi_library_file_name2 = f'{capi_library_name}.lib'
+elif platform.system() == 'Darwin':
     if is_x64:
-        capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls'
-        capi_library_dir = capi_root_dir / 'bin' / 'x64'
+        capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls'
+        capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64'
         capi_library_name = 'DXFeed_64'
+        capi_library_file_name = f'lib{capi_library_name}.dylib'
     else:
         raise Exception('Unsupported platform')
 else:
     if is_x64:
-        capi_root_dir = capi_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls'
-        capi_library_dir = capi_root_dir / 'bin' / 'x64'
+        capi_extracted_root_dir = capi_extracted_root_dir / f'DXFeedAll-{capi_version}-x64-no-tls'
+        capi_extracted_library_dir = capi_extracted_root_dir / 'bin' / 'x64'
         capi_library_name = 'DXFeed_64'
+        capi_library_file_name = f'lib{capi_library_name}.so'
     else:
         raise Exception('Unsupported platform')
 
-if current_os == 'windows':
-    runtime_library_dirs = []
+if not os.path.exists(capi_include_dir):
+    shutil.copytree(capi_extracted_root_dir / 'include', capi_include_dir)
+if not os.path.exists(capi_bin_dir / capi_library_file_name):
+    #    os.makedirs(capi_bin_dir)
+    shutil.copyfile(capi_extracted_library_dir / capi_library_file_name, capi_bin_dir / capi_library_file_name)
+if platform.system() == 'Windows':
+    # noinspection PyUnboundLocalVariable
+    if not os.path.exists(capi_bin_dir / capi_library_file_name2):
+        shutil.copyfile(capi_extracted_library_dir / capi_library_file_name2, capi_bin_dir / capi_library_file_name2)
+
+if platform.system() == 'Windows':
+    runtime_library_dirs = None
 else:
-    runtime_library_dirs = [str(capi_library_dir)]
+    runtime_library_dirs = [str(capi_bin_dir)]
 
-capi_include_dirs = [str(capi_root_dir / 'include'), str(capi_root_dir / 'src')]
+capi_include_dirs = [str(capi_include_dir)]
 
 libs = [capi_library_name]
 if platform.system() == 'Windows':
@@ -82,7 +104,7 @@
                         include_dirs=capi_include_dirs),
               Extension('dxfeed.core.listeners.listener', ['dxfeed/core/listeners/listener.' + ext],
                         include_dirs=capi_include_dirs),
-              Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_library_dir)],
+              Extension('dxfeed.core.DXFeedPy', ['dxfeed/core/DXFeedPy.' + ext], library_dirs=[str(capi_bin_dir)],
                         runtime_library_dirs=runtime_library_dirs,
                         libraries=libs,
                         include_dirs=capi_include_dirs)]
@@ -115,3 +137,7 @@ def build_extensions():
     build_ext_cmd.ensure_finalized()
     build_ext_cmd.inplace = 1
     build_ext_cmd.run()
+
+    print(f'Removing the "{path_to_extract}"')
+    if os.path.exists(path_to_extract):
+        shutil.rmtree(path_to_extract)

clear.py

@@ -0,0 +1,23 @@
+import glob
+import os
+import shutil
+
+c_files = glob.glob('dxfeed/core/**/*.c', recursive=True)
+cpp_files = glob.glob('dxfeed/core/**/*.cpp', recursive=True)
+pyd_files = glob.glob('dxfeed/core/**/*.pyd', recursive=True)
+so_files = glob.glob('dxfeed/core/**/*.so', recursive=True)
+dll_files = glob.glob('dxfeed/core/**/*.dll', recursive=True)
+lib_files = glob.glob('dxfeed/core/**/*.lib', recursive=True)
+dylib_files = glob.glob('dxfeed/core/**/*.dylib', recursive=True)
+
+for file_path in c_files + cpp_files + pyd_files + so_files + dll_files + lib_files + dylib_files:
+    os.remove(file_path)
+
+if os.path.exists('dxfeed/tmp'):
+    shutil.rmtree('dxfeed/tmp')
+
+if os.path.exists('dxfeed/dxfeed-c-api/bin'):
+    shutil.rmtree('dxfeed/dxfeed-c-api/bin')
+
+if os.path.exists('dxfeed/dxfeed-c-api/include'):
+    shutil.rmtree('dxfeed/dxfeed-c-api/include')

pyproject.toml

@@ -17,38 +17,37 @@ classifiers = ['Development Status :: 4 - Beta',
                'Natural Language :: English']
 
 include = ['LICENSE']
-#include = ['LICENSE', 'dxfeed/**/*.c']
-exclude = ['dxfeed/dxfeed-c-api*', 'dxfeed/tmp*']
+exclude = ['dxfeed/tmp*']
 
 [tool.poetry.urls]
 'Source code' = 'https://github.com/dxFeed/dxfeed-python-api'
 'Documentation' = 'https://dxfeed.readthedocs.io/en/latest/'
 
 [tool.poetry.dependencies]
 python = '^3.6'
-pandas = '^1.0.0'
-toml = '^0.10.0'
+pandas = [{version = '^1.0.0', python = '^3.6.1'}, {version = '^1.3.2', python = '^3.7.1'}]
+toml = '^0.10.2'
 # The following dependencies are used for docs generation when installed as extras
 # (e.g. pip install nornir[docs])
 # Currently they have to be specified as non-dev dependencies with optional = true
 # Refer to: https://github.com/sdispater/poetry/issues/129
 # Issue to watch for: https://github.com/python-poetry/poetry/issues/1644
 jupyter = { version = '^1.0.0', optional = true }
-cython = { version = '^0.29.13', optional = true }
+cython = { version = '^0.29.14', optional = true }
 
 [tool.poetry.dev-dependencies]
-cython = '^0.29.13'
-taskipy = '^1.1.3'
-pytest = '^5.3.5'
-sphinx = '^2.4.4'
-sphinx_rtd_theme = '^0.4.3'
-pygments = '^2.6.1'
+cython = '^0.29.14'
+taskipy = '^1.8.1'
+pytest = '^6.2.4'
+sphinx = '^4.1.2'
+sphinx_rtd_theme = '^0.5.2'
+pygments = '^2.10'
 
 [tool.poetry.extras]
 docs = ['jupyter', 'cython']
 
 [tool.taskipy.tasks]
-clear = 'find dxfeed/core -type f \( -iname *.c -o -iname *.cpp -o -iname *.pyd -o -iname *.so \) -delete'
+clear = 'python clear.py'
 build = 'python build.py && poetry build'
 build_inplace = 'python -c "from build import *; build_extensions()"'
 html_docs = 'make html -C docs'

tests/test_dxfeedpy.py

@@ -90,7 +90,7 @@ def test_symbol_addition(connection):
     dxc.dxf_add_symbols(sc=sub, symbols=symbols)
     actual_symbols = dxc.dxf_get_symbols(sc=sub)
     dxc.dxf_close_subscription(sub)
-    assert symbols == actual_symbols
+    assert set(symbols) == set(actual_symbols)
 
 
 def test_symbol_deletion(connection):
@@ -110,7 +110,7 @@ def test_wrong_symbol_types_ignored(connection):
     dxc.dxf_add_symbols(sc=sub, symbols=symbols + [1, 5.0, [], True, {}, ()])
     actual_symbols = dxc.dxf_get_symbols(sc=sub)
     dxc.dxf_close_subscription(sub)
-    assert symbols == actual_symbols
+    assert set(symbols) == set(actual_symbols)
 
 
 def test_symbol_clearing(connection):
@@ -155,4 +155,4 @@ def test_double_event_handler_attachment(connection):
     dxc.dxf_add_symbols(sub, symbols)
     sub.set_event_handler(handler2)
     assert sub.get_event_handler() is handler2
-    assert dxc.dxf_get_symbols(sub) == symbols
+    assert set(dxc.dxf_get_symbols(sub)) == set(symbols)

tests/test_subscription_class.py

@@ -42,24 +42,24 @@ def test_subscription_timed_event_type_property(subscription_timed):
 
 def test_subscription_event_type_property(subscription):
     subscription = subscription.add_symbols(ValueStorage.symbols)
-    assert subscription.symbols == ValueStorage.symbols
+    assert set(subscription.symbols) == set(ValueStorage.symbols)
 
 
 def test_subscription_timed_event_type_property(subscription_timed):
     subscription = subscription_timed.add_symbols(ValueStorage.symbols)
-    assert subscription.symbols == ValueStorage.symbols
+    assert set(subscription.symbols) == set(ValueStorage.symbols)
 
 
 def test_remove_symbol(subscription):
     subscription = subscription.add_symbols(ValueStorage.symbols) \
                                .remove_symbols([ValueStorage.symbols[0]])
-    assert subscription.symbols == ValueStorage.symbols[1:]
+    assert set(subscription.symbols) == set(ValueStorage.symbols[1:])
 
 
 def test_remove_symbol_timed(subscription_timed):
     subscription_timed = subscription_timed.add_symbols(ValueStorage.symbols) \
                                            .remove_symbols([ValueStorage.symbols[0]])
-    assert subscription_timed.symbols == ValueStorage.symbols[1:]
+    assert set(subscription_timed.symbols) == set(ValueStorage.symbols[1:])
 
 
 def test_remove_all_symbols(subscription):