From 6946d8d041c444a7b69ba14fa9336c2cd1d51be7 Mon Sep 17 00:00:00 2001 From: HansBug Date: Sun, 19 Jun 2022 20:37:36 +0800 Subject: [PATCH 1/3] test(hansbug): try add cibuildwheel && first complete the try_release part && move it out && remove benchmark from unittest --- .github/workflows/release_test.yml | 39 +++++++++++ .github/workflows/run.yml | 2 +- .github/workflows/test.yml | 109 ++--------------------------- pyproject.toml | 41 ++++++++++- requirements-test.txt | 1 + test/entry/cli/test_graph.py | 8 +++ 6 files changed, 95 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/release_test.yml diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml new file mode 100644 index 0000000000..eaf0aeb194 --- /dev/null +++ b/.github/workflows/release_test.yml @@ -0,0 +1,39 @@ +name: Release Test + +on: workflow_dispatch + +jobs: + release_test: + name: Try build the release + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-18.04' +# - 'windows-2019' + - 'macos-10.15' + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 20 + + - name: Set up QEMU + if: ${{ runner.os == 'Linux' }} + uses: docker/setup-qemu-action@v1 + with: + platforms: all + - name: Build wheels + uses: pypa/cibuildwheel@v2.7.0 + + - name: Show the buildings + shell: bash + run: | + ls -al ./wheelhouse + - name: Upload packed files to artifacts + uses: actions/upload-artifact@v2 + with: + name: build-artifacts-${{ matrix.os }} + path: ./wheelhouse/* diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 0f89a79d02..22035c855c 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -2,7 +2,7 @@ name: Code Script Run on: push: - branches: [ main, 'dev/*', 'run/*' ] + branches: [ 'run/*' ] jobs: unittest: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e3fc3cbb5..309b23aa22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,12 @@ name: Code Test on: - - push + push: + branches: + - main + - dev/* + - fix/* + - test/* jobs: unittest: @@ -113,105 +118,3 @@ jobs: flags: unittests name: codecov-umbrella fail_ci_if_error: false - - benchmark: - name: Benchmark Test - runs-on: ${{ matrix.os }} - if: "!contains(github.event.head_commit.message, 'ci skip')" - strategy: - fail-fast: false - matrix: - os: - - 'ubuntu-18.04' - python-version: - - '3.6' - - '3.7' - - '3.8' - - '3.9' - - '3.10' - - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 20 - - name: Set up system dependences on linux - if: ${{ runner.os == 'Linux' }} - run: | - sudo apt-get update - sudo apt-get install -y tree cloc wget curl make graphviz - sudo apt-get install -y libxml2-dev libxslt-dev python-dev # need by pypy3 - dot -V - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade flake8 setuptools wheel twine - pip install -r requirements.txt - pip install -r requirements-build.txt - pip install -r requirements-test.txt - pip install -r requirements-benchmark.txt - ./install_test.sh - - name: Test the basic environment - run: | - python -V - pip --version - pip list - tree . - cloc treevalue - cloc test - - name: Run benckmark - env: - CI: 'true' - run: | - make clean build benchmark compare - - try_release: - name: Try build the release - runs-on: ${{ matrix.os }} - if: "!contains(github.event.head_commit.message, 'ci skip')" - strategy: - fail-fast: false - matrix: - os: - - 'ubuntu-18.04' - python-version: - - '3.6' - - '3.7' - - '3.8' - - '3.9' - - '3.10' - - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - fetch-depth: 20 - - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Set up python dependences - run: | - pip install --upgrade pip - pip install --upgrade flake8 setuptools wheel twine - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f requirements-build.txt ]; then pip install -r requirements-build.txt; fi - if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi - pip install --upgrade build - - name: Build packages - run: | - make package - ls -al dist - - name: Get package version - run: | - make build - python -c 'from treevalue.config.meta import __VERSION__;print(__VERSION__)' - echo "PACKAGE_VERSION=$(python -c 'from treevalue.config.meta import __VERSION__;print(__VERSION__)')" >> $GITHUB_ENV - - uses: actions/upload-artifact@v2 - with: - name: treevalue-v${{ env.PACKAGE_VERSION }}-${{ matrix.os }}-${{ matrix.python-version }} - path: dist diff --git a/pyproject.toml b/pyproject.toml index bc2be6730f..1be3c041c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,41 @@ +[project] +requires-python = ">=3.6" + [build-system] -requires = ["setuptools", "wheel", "Cython"] +requires = [ + "setuptools>=42", + "wheel", + "Cython", +] + +[tool.cibuildwheel] +skip = ["pp*"] # Do not build for PyPy + + +## macOS build configuration +[tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] # Build for x86_64 and arm64 +before-test = [# Unittest for linux + "pip install -r {project}/requirements-test.txt", +] +test-command = [ + "cp -rf {project}/test test", + "cp {project}/pytest.ini pytest.ini", + "pytest test -sv -m unittest --log-level=DEBUG", + "rm -rf test", +] + + +## Linux build configuration +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64"] # Build for x86_64 and arm64 +skip = ["pp* *musllinux*"] # dependencies do not build for musl +before-test = [# Unittest for linux + "pip install -r {project}/requirements-test.txt", +] +test-command = [ + "cp -rf {project}/test test", + "cp {project}/pytest.ini pytest.ini", + "pytest test -sv -m unittest --log-level=DEBUG", + "rm -rf test", +] diff --git a/requirements-test.txt b/requirements-test.txt index c405ace95b..2a5797e35b 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -9,6 +9,7 @@ pytest-rerunfailures~=10.2 pytest-timeout~=2.0.2 pytest-benchmark~=3.4.0 testtools>=2 +hbutils>=0.6.13 setuptools<=59.5.0 numpy>=1.10 torch>=1.1.0 diff --git a/test/entry/cli/test_graph.py b/test/entry/cli/test_graph.py index 22a18f7856..9fe6bdfec5 100644 --- a/test/entry/cli/test_graph.py +++ b/test/entry/cli/test_graph.py @@ -1,11 +1,13 @@ import os import pathlib import pickle +import unittest import warnings import zlib import pytest from click.testing import CliRunner +from hbutils.testing import cmdv from treevalue import FastTreeValue, dump, graphics from treevalue.entry.cli import treevalue_cli @@ -24,6 +26,7 @@ @pytest.mark.unittest class TestEntryCliGraph: + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_simple_code_graph(self): runner = CliRunner() with runner.isolated_filesystem(): @@ -54,6 +57,7 @@ def test_simple_code_graph_to_stdout(self): assert not os.path.exists('test_graph.gv') assert len(result.output) <= 2500 + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_simple_code_multiple_graph(self): runner = CliRunner() with runner.isolated_filesystem(): @@ -76,6 +80,7 @@ def test_simple_code_multiple_graph(self): assert os.path.exists('test_graph.svg') assert os.path.getsize('test_graph.svg') <= 17500 + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_simple_binary_graph(self): runner = CliRunner() with runner.isolated_filesystem(): @@ -134,6 +139,7 @@ def test_simple_binary_graph(self): assert os.path.exists('test_graph.svg') assert os.path.getsize('test_graph.svg') <= 6500 + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_duplicates(self): runner = CliRunner() with runner.isolated_filesystem(): @@ -162,6 +168,7 @@ def test_duplicates(self): shutil.copy('test_graph.svg', os.path.join(_p, 'test_graph.svg')) assert os.path.getsize('test_graph.svg') <= 11000 + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_graph(self): runner = CliRunner() with runner.isolated_filesystem(): @@ -203,6 +210,7 @@ def test_cfg(self): assert result.exit_code != 0 assert "Configuration should be KEY=VALUE, but 'bgcolor#ffffff00' found." in result.output + @unittest.skipUnless(cmdv('dot'), 'Dot installed only') def test_file_with_invalid_permission(self): runner = CliRunner() with runner.isolated_filesystem(): From 59a2be22e0d3b32da65732e5824535ea11d13d43 Mon Sep 17 00:00:00 2001 From: HansBug Date: Mon, 20 Jun 2022 13:41:06 +0800 Subject: [PATCH 2/3] test(hansbug): split build all the wheels --- .github/workflows/release.yml | 21 ++++---- .github/workflows/release_test.yml | 83 ++++++++++++++++++++++++++++-- Makefile | 5 +- pyproject.toml | 3 -- 4 files changed, 96 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b15d802502..8f0f729a2f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,8 +5,8 @@ on: types: [ published ] jobs: - release: - name: Publish to official pypi + source_release: + name: Publish source package to official pypi runs-on: ${{ matrix.os }} if: ${{ github.repository == 'opendilab/treevalue' }} strategy: @@ -14,11 +14,7 @@ jobs: os: - 'ubuntu-18.04' python-version: - - '3.6' - - '3.7' - '3.8' - - '3.9' - - '3.10' steps: - name: Checkout code @@ -39,10 +35,17 @@ jobs: pip install --upgrade build - name: Build packages run: | - make package + make zip - name: Publish distribution 📦 to real PyPI uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.PYPI_PASSWORD }} - skip_existing: true - verbose: true \ No newline at end of file + verbose: true + - name: Upload distribution 📦 to github release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist/* + tag: ${{ github.ref }} + overwrite: true + file_glob: true diff --git a/.github/workflows/release_test.yml b/.github/workflows/release_test.yml index eaf0aeb194..0fcbe57e25 100644 --- a/.github/workflows/release_test.yml +++ b/.github/workflows/release_test.yml @@ -3,8 +3,46 @@ name: Release Test on: workflow_dispatch jobs: - release_test: - name: Try build the release + source_release: + name: Try package the source + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-18.04' + python-version: + - '3.8' + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 20 + - name: Set up python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Set up python dependences + run: | + pip install --upgrade pip + pip install --upgrade flake8 setuptools wheel twine + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-build.txt ]; then pip install -r requirements-build.txt; fi + if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi + pip install --upgrade build + - name: Build packages + run: | + make zip + ls -al dist + - name: Upload packed files to artifacts + uses: actions/upload-artifact@v2 + with: + name: build-artifacts-source-pack + path: ./dist/* + + wheel_release: + name: Try build the wheels runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -13,6 +51,41 @@ jobs: - 'ubuntu-18.04' # - 'windows-2019' - 'macos-10.15' + python: + - '3.6' + - '3.7' + - '3.8' + - '3.9' + - '3.10' + architecture: + - x86_64 + - arm64 + - aarch64 + - x86 + - AMD64 + exclude: + - os: ubuntu-18.04 + architecture: arm64 + - os: ubuntu-18.04 + architecture: x86 + - os: ubuntu-18.04 + architecture: AMD64 + - os: windows-2019 + architecture: x86_64 + - os: windows-2019 + architecture: arm64 + - os: windows-2019 + architecture: aarch64 + - os: macos-10.15 + architecture: aarch64 + - os: macos-10.15 + architecture: x86 + - os: macos-10.15 + architecture: AMD64 + - python: '3.6' + architecture: arm64 + - python: '3.7' + architecture: arm64 steps: - name: Checkout code @@ -27,6 +100,10 @@ jobs: platforms: all - name: Build wheels uses: pypa/cibuildwheel@v2.7.0 + env: + CIBW_SKIP: "pp* *musllinux*" + CIBW_ARCHS: ${{ matrix.architecture }} + CIBW_PROJECT_REQUIRES_PYTHON: ~=${{ matrix.python }}.0 - name: Show the buildings shell: bash @@ -35,5 +112,5 @@ jobs: - name: Upload packed files to artifacts uses: actions/upload-artifact@v2 with: - name: build-artifacts-${{ matrix.os }} + name: build-artifacts-${{ runner.os }}-cp${{ matrix.python }}-${{ matrix.architecture }} path: ./wheelhouse/* diff --git a/Makefile b/Makefile index de3c4417a2..fb9a8f84c4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: docs test unittest build clean benchmark +.PHONY: docs test unittest build clean benchmark zip PYTHON := $(shell which python) @@ -24,6 +24,9 @@ build: $(PYTHON) setup.py build_ext --inplace \ $(if ${LINETRACE},--define CYTHON_TRACE,) +zip: + $(PYTHON) -m build --sdist --outdir ${DIST_DIR} + package: $(PYTHON) -m build --sdist --wheel --outdir ${DIST_DIR} for whl in `ls ${DIST_DIR}/*.whl`; do \ diff --git a/pyproject.toml b/pyproject.toml index 1be3c041c4..f0742e0f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,3 @@ -[project] -requires-python = ">=3.6" - [build-system] requires = [ "setuptools>=42", From 634091bdd9f5e16c4bd008f0eb29e2fad1533399 Mon Sep 17 00:00:00 2001 From: HansBug Date: Mon, 20 Jun 2022 14:50:36 +0800 Subject: [PATCH 3/3] test(hansbug): add wheel_release --- .github/workflows/release.yml | 84 ++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f0f729a2f..28e04ad8d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,5 +47,87 @@ jobs: repo_token: ${{ secrets.GITHUB_TOKEN }} file: dist/* tag: ${{ github.ref }} - overwrite: true + file_glob: true + + + wheel_release: + name: Try build the wheels + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - 'ubuntu-18.04' +# - 'windows-2019' + - 'macos-10.15' + python: + - '3.6' + - '3.7' + - '3.8' + - '3.9' + - '3.10' + architecture: + - x86_64 + - arm64 + - aarch64 + - x86 + - AMD64 + exclude: + - os: ubuntu-18.04 + architecture: arm64 + - os: ubuntu-18.04 + architecture: x86 + - os: ubuntu-18.04 + architecture: AMD64 + - os: windows-2019 + architecture: x86_64 + - os: windows-2019 + architecture: arm64 + - os: windows-2019 + architecture: aarch64 + - os: macos-10.15 + architecture: aarch64 + - os: macos-10.15 + architecture: x86 + - os: macos-10.15 + architecture: AMD64 + - python: '3.6' + architecture: arm64 + - python: '3.7' + architecture: arm64 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 20 + + - name: Set up QEMU + if: ${{ runner.os == 'Linux' }} + uses: docker/setup-qemu-action@v1 + with: + platforms: all + - name: Build wheels + uses: pypa/cibuildwheel@v2.7.0 + env: + CIBW_SKIP: "pp* *musllinux*" + CIBW_ARCHS: ${{ matrix.architecture }} + CIBW_PROJECT_REQUIRES_PYTHON: ~=${{ matrix.python }}.0 + + - name: Show the buildings + shell: bash + run: | + ls -al ./wheelhouse + - name: Publish distribution 📦 to real PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_PASSWORD }} + verbose: true + packages_dir: wheelhouse/ + - name: Upload distribution 📦 to github release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: wheelhouse/* + tag: ${{ github.ref }} file_glob: true