From 5ba14e2335196f4aac059c476a07070ef69a54f2 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 09:52:06 -0400 Subject: [PATCH 1/8] move SDK download to setup --- setup.py | 74 ++++++++++++++++++++++++++++++++++++++++---- tools/get_zitilib.py | 74 -------------------------------------------- 2 files changed, 68 insertions(+), 80 deletions(-) delete mode 100644 tools/get_zitilib.py diff --git a/setup.py b/setup.py index 3680cc5..bdd3f4b 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import setuptools from setuptools import setup +from setuptools.command.build_ext import build_ext +from setuptools.extension import Extension + import versioneer # read the contents of your README file @@ -21,13 +23,73 @@ this_directory = Path(__file__).parent long_description = (this_directory / "README.md").read_text() +ZITI_SDK_BASE = 'https://github.com/openziti/ziti-sdk-c/releases/download' + + +class GetZitilib(build_ext): + + def build_extension(self, ext) -> None: + import platform + ziti_ver = self.get_sdk_version() + osname = platform.system() + sdk_distro = self.download_sdk(ziti_ver, osname, platform.machine()) + self.extract_zitilib(sdk_distro, self.get_lib_name(osname), self.build_lib) + + def get_lib_name(self, osname): + osname = osname.capitalize() + if osname == 'Linux': + libname = 'libziti.so' + elif osname == 'Darwin': + libname = 'libziti.dylib' + elif osname == 'Windows': + libname = 'ziti.dll' + else: + raise RuntimeError(f"Unsupported platform[{osname}]") + return libname + + def get_sdk_version(self): + opts = self.distribution.get_option_dict('openziti') + _, ver = opts['ziti_sdk_version'] + return ver + + def extract_zitilib(self, distro, libname, target): + import zipfile + from io import BytesIO + with zipfile.ZipFile(BytesIO(distro)) as zipf: + return zipf.extract(member=f'lib/{libname}', path=f'{target}/openziti') + + def download_sdk(self, version, osname, arch): + from urllib.error import HTTPError + from urllib.request import Request, urlopen + filename = f'{ZITI_SDK_BASE}/{version}/ziti-sdk-{version}-{osname}-{arch}.zip' + headers = {} + req = Request(url=filename, headers=headers) + try: + with urlopen(req) as response: + length = response.getheader('content-length') + if response.status != 200: + raise Exception(f'Could not download "{filename}"') + return None + + print(f"Downloading {length} from {filename}") + return response.read() + except HTTPError: + raise + +class ZitilibExt(Extension): + def __init__(self, name, sourcedir=''): + Extension.__init__(self, name, sources=[]) + + +cmds = dict(build_ext=GetZitilib) +cmds = versioneer.get_cmdclass(cmds) + setup( version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + cmdclass=cmds, + ext_modules=[ + ZitilibExt('_get_ziti_lib'), + ], packages=['openziti'], - include_package_data=True, - package_data={ - "openziti": ["lib/*"], - }, ) \ No newline at end of file diff --git a/tools/get_zitilib.py b/tools/get_zitilib.py deleted file mode 100644 index afb7000..0000000 --- a/tools/get_zitilib.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2022. NetFoundry, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import zipfile -from configparser import ConfigParser -from io import BytesIO -from os.path import dirname -from urllib.error import HTTPError -from urllib.request import Request, urlopen - -ZITI_SDK_BASE = 'https://github.com/openziti/ziti-sdk-c/releases/download' - - -def download_sdk(version): - filename = f'{ZITI_SDK_BASE}/{version}/ziti-sdk-{version}-{osname}-{arch}.zip' - headers = {} - req = Request(url=filename, headers=headers) - try: - with urlopen(req) as response: - length = response.getheader('content-length') - if response.status != 200: - print(f'Could not download "{filename}"', file=sys.stderr) - return None - - print(f"Downloading {length} from {filename}", file=sys.stderr) - return response.read() - except HTTPError: - print(f'Could not download "{filename}"', file=sys.stderr) - raise - - -def extract(data, libname): - with zipfile.ZipFile(BytesIO(data)) as zipf: - return zipf.extract(member=f'lib/{libname}', path='src/openziti/') - - -def get_sdk_version(): - cfgfile = f'{dirname(__file__)}/../setup.cfg' - parser = ConfigParser() - parser.read(cfgfile) - return dict(parser.items('openziti'))['ziti_sdk_version'] - - -if __name__ == '__main__': - import platform - sdk_version = get_sdk_version() - osname = platform.system() - arch = platform.machine() - print(f'platform={osname}-{arch}') - - osname = osname.capitalize() - if osname == 'Linux': - LIBNAME = 'libziti.so' - elif osname == 'Darwin': - LIBNAME = 'libziti.dylib' - elif osname == 'Windows': - LIBNAME = 'ziti.dll' - else: - raise RuntimeError("Unsupported platform/arch") - - d = download_sdk(version=sdk_version) - libfile = extract(d, LIBNAME) From 7395f176002b3196d1d3c6ccfc9958341e5af909 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 09:53:23 -0400 Subject: [PATCH 2/8] change CI to build sdist --- .github/workflows/wheels.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 112825f..75df538 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -28,7 +28,7 @@ jobs: python-version: '3.x' - name: Install Python Tools - run: python -m pip install -U pip setuptools wheel auditwheel + run: python -m pip install -U pip setuptools - name: Get Ziti SDK C run: | @@ -47,20 +47,15 @@ jobs: pip install . python -m unittest tests/ziti_tests.py - - name: Build wheels + - name: Build distro run: | - pip wheel . -w ./wheelhouse --build-option --plat-name=${{ matrix.spec.target }} - - - name: Audit wheel - if: startsWith(matrix.spec.name, 'linux') - run: | - auditwheel show ./wheelhouse/* - auditwheel repair --plat=${{ matrix.spec.target }} ./wheelhouse/* + python setup.py sdist - uses: actions/upload-artifact@v3 + if: startsWith(matrix.spec.name, 'linux') with: - name: ${{ matrix.spec.name }} - path: ./wheelhouse/*.whl + name: openziiti-sdist + path: ./dist/* publish: runs-on: ubuntu-18.04 From 50c7fa5124643067da0176f34b5deaa01567dab5 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 09:55:40 -0400 Subject: [PATCH 3/8] bump ziti-sdk@0.29.6 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index cad9802..c8e6c0a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,4 +38,4 @@ tag_prefix = v parentdir_prefix = openziti- [openziti] -ziti_sdk_version = 0.29.3 \ No newline at end of file +ziti_sdk_version = 0.29.6 \ No newline at end of file From 725d234a841e5c9e66adf4f6aaed0b887d962d1a Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 09:57:03 -0400 Subject: [PATCH 4/8] drop download step --- .github/workflows/wheels.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 75df538..73757e4 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -30,10 +30,6 @@ jobs: - name: Install Python Tools run: python -m pip install -U pip setuptools - - name: Get Ziti SDK C - run: | - python tools/get_zitilib.py - - name: Run Integration Tests if: ${{ env.HAVE_TEST_ID == 'true' }} env: From 6a56510652a56acd55d9ae3378f0093a239c5044 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 09:59:48 -0400 Subject: [PATCH 5/8] switch from old ubuntu hosts --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 73757e4..ec541c8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -54,7 +54,7 @@ jobs: path: ./dist/* publish: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 needs: [ build_wheels ] steps: - name: Download artifacts From 065b08964d24af856fe83941f71f64fa57e440b9 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 12:17:38 -0400 Subject: [PATCH 6/8] adjust SDK bundle selection --- setup.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index bdd3f4b..0ed16bb 100644 --- a/setup.py +++ b/setup.py @@ -31,21 +31,26 @@ class GetZitilib(build_ext): def build_extension(self, ext) -> None: import platform ziti_ver = self.get_sdk_version() - osname = platform.system() - sdk_distro = self.download_sdk(ziti_ver, osname, platform.machine()) - self.extract_zitilib(sdk_distro, self.get_lib_name(osname), self.build_lib) + osname, arch, libname = self.get_platform() + sdk_distro = self.download_sdk(ziti_ver, osname, arch) + self.extract_zitilib(sdk_distro, libname, self.build_lib) - def get_lib_name(self, osname): - osname = osname.capitalize() + def get_platform(self): + import platform + osname = platform.system() + mach = platform.machine() + arch, _ = platform.architecture() if osname == 'Linux': - libname = 'libziti.so' + if mach.startswith('arm'): + if arch == '32bit': + mach = 'arm' + elif arch == '64bit': + mach = 'arm64' + return osname, mach, 'libziti.so' elif osname == 'Darwin': - libname = 'libziti.dylib' + return osname, mach, 'libziti.dylib' elif osname == 'Windows': - libname = 'ziti.dll' - else: - raise RuntimeError(f"Unsupported platform[{osname}]") - return libname + return osname, mach, 'ziti.dll' def get_sdk_version(self): opts = self.distribution.get_option_dict('openziti') From 79021e311184b8ded81d03bee48d2fe71d71e786 Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 13:15:31 -0400 Subject: [PATCH 7/8] cleanup based on comments --- setup.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 0ed16bb..e28fc65 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import platform from setuptools import setup from setuptools.command.build_ext import build_ext from setuptools.extension import Extension +from urllib.request import Request, urlopen +import zipfile +from io import BytesIO import versioneer @@ -29,14 +33,12 @@ class GetZitilib(build_ext): def build_extension(self, ext) -> None: - import platform ziti_ver = self.get_sdk_version() osname, arch, libname = self.get_platform() sdk_distro = self.download_sdk(ziti_ver, osname, arch) self.extract_zitilib(sdk_distro, libname, self.build_lib) def get_platform(self): - import platform osname = platform.system() mach = platform.machine() arch, _ = platform.architecture() @@ -58,28 +60,19 @@ def get_sdk_version(self): return ver def extract_zitilib(self, distro, libname, target): - import zipfile - from io import BytesIO with zipfile.ZipFile(BytesIO(distro)) as zipf: return zipf.extract(member=f'lib/{libname}', path=f'{target}/openziti') def download_sdk(self, version, osname, arch): - from urllib.error import HTTPError - from urllib.request import Request, urlopen filename = f'{ZITI_SDK_BASE}/{version}/ziti-sdk-{version}-{osname}-{arch}.zip' headers = {} req = Request(url=filename, headers=headers) - try: - with urlopen(req) as response: - length = response.getheader('content-length') - if response.status != 200: - raise Exception(f'Could not download "{filename}"') - return None - - print(f"Downloading {length} from {filename}") - return response.read() - except HTTPError: - raise + with urlopen(req) as response: + length = response.getheader('content-length') + if response.status != 200: + raise Exception(f'Could not download "{filename}"') + print(f"Downloading {length} from {filename}") + return response.read() class ZitilibExt(Extension): def __init__(self, name, sourcedir=''): From 4bccffc0f224812be9f89f63c5826304e9163cfd Mon Sep 17 00:00:00 2001 From: Eugene K Date: Mon, 22 Aug 2022 13:16:40 -0400 Subject: [PATCH 8/8] cleanup based on comments --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e28fc65..b4e9a4a 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ def get_platform(self): osname = platform.system() mach = platform.machine() arch, _ = platform.architecture() + if osname == 'Linux': if mach.startswith('arm'): if arch == '32bit': @@ -49,9 +50,11 @@ def get_platform(self): elif arch == '64bit': mach = 'arm64' return osname, mach, 'libziti.so' - elif osname == 'Darwin': + + if osname == 'Darwin': return osname, mach, 'libziti.dylib' - elif osname == 'Windows': + + if osname == 'Windows': return osname, mach, 'ziti.dll' def get_sdk_version(self):