diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml deleted file mode 100644 index c5e2ebf3..00000000 --- a/.github/actions/install-dependencies/action.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: 'Install pyosmium dependencies' - -inputs: - version: - description: "Which version to install (release or develop)" - required: true - -runs: - using: "composite" - - steps: - - name: Install package dependencies (develop) - run: | - git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium - git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero - git clone --quiet https://github.com/pybind/pybind11.git contrib/pybind11 - shell: bash - if: ${{ inputs.version == 'develop' }} - - - name: Install package dependencies (release) - run: | - export PATH=$PATH:/c/msys64/usr/bin - OSMIUM_VER=`grep libosmium_version src/osmium/version.py | sed "s:.*= '::;s:'.*::"` - PROTOZERO_VER=`grep protozero_version src/osmium/version.py | sed "s:.*= '::;s:'.*::"` - PYBIND_VER=`grep pybind11_version src/osmium/version.py | sed "s:.*= '::;s:'.*::"` - wget -O - https://github.com/osmcode/libosmium/archive/v$OSMIUM_VER.tar.gz | tar xz --one-top-level=contrib/libosmium --strip-components=1 - wget -O - https://github.com/mapbox/protozero/archive/v$PROTOZERO_VER.tar.gz | tar xz --one-top-level=contrib/protozero --strip-components=1 - wget -O - https://github.com/pybind/pybind11/archive/v$PYBIND_VER.tar.gz | tar xz --one-top-level=contrib/pybind11 --strip-components=1 - shell: bash - if: ${{ inputs.version == 'release' }} diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index b155b131..95977576 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -7,62 +7,51 @@ on: jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-2019, macos-13, macos-14] - arch: [native] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-latest] + vcpkg: [none] include: - - os: ubuntu-20.04 - arch: aarch64 + - os: windows-latest + vcpkg: 'C:/vcpkg/installed/x64-windows' + - os: windows-11-arm + vcpkg: 'C:/vcpkg/installed/arm64-windows' steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: pybind/pybind11 - ref: v2.11.1 - path: contrib/pybind11 - - - uses: actions/checkout@v4 - with: - repository: mapbox/protozero - ref: v1.7.1 - path: contrib/protozero - - - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - repository: osmcode/libosmium - ref: v2.20.0 - path: contrib/libosmium + python-version: '3.13' - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - if: ${{ matrix.arch == 'aarch64' }} + - name: Install requirements + run: python -m pip install cibuildwheel build - - uses: actions/setup-python@v5 - with: - python-version: '3.8' + - name: Get dependencies via sdist + run: python -m build --sdist - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + run: | + python -m cibuildwheel --output-dir wheelhouse + shell: bash env: - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_SKIP: "pp* *musllinux* cp37-macosx_* {cp37,cp38}-*linux_aarch64" - CIBW_TEST_REQUIRES: pytest pytest-httpserver shapely - CIBW_TEST_REQUIRES_LINUX: urllib3<2.0 pytest pytest-httpserver shapely + CIBW_ENABLE: cpython-freethreading + CIBW_ARCHS: "native" + CIBW_SKIP: "*musllinux*" + CIBW_TEST_REQUIRES: pytest pytest-httpserver CIBW_TEST_COMMAND: pytest {project}/test CIBW_BUILD_FRONTEND: build CIBW_BEFORE_BUILD_LINUX: yum install -y expat-devel boost-devel zlib-devel bzip2-devel lz4-devel CIBW_BEFORE_BUILD_MACOS: brew install boost - CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows + CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 SKBUILD_CMAKE_ARGS=-DWITH_LZ4=OFF + CIBW_BEFORE_BUILD_WINDOWS: vcpkg install bzip2 expat zlib boost-variant boost-iterator lz4 CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake"' - CIBW_ENVIRONMENT_MACOS: CMAKE_WITH_LZ4=OFF MACOSX_DEPLOYMENT_TARGET=11.0 + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: 'pipx run delvewheel repair --add-path ${{ matrix.vcpkg }}/bin/ --add-path ${{ matrix.vcpkg }}/debug/bin -w {dest_dir} {wheel}' - uses: actions/upload-artifact@v4 with: - name: pyosmium-wheels-${{ matrix.os }}-${{ matrix.arch }} + name: pyosmium-wheels-${{ matrix.os }} path: ./wheelhouse/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74f05076..fb56ee28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,6 +268,7 @@ jobs: runs-on: ${{ matrix.platform }} strategy: + fail-fast: false matrix: compiler: [gcc-old, clang-old, gcc, clang, macos-intel, macos-arm] include: @@ -317,7 +318,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install prerequisites + - name: Clean up build environment run: | # Workaround for github/brew problem. Python is already install # on the Github action runner and then homebrew comes along... @@ -325,39 +326,52 @@ jobs: rm -f /usr/local/bin/2to3* /usr/local/bin/idle3* /usr/local/bin/pydoc3* /usr/local/bin/python3* if: ${{ matrix.flavour == 'macos' }} - - uses: ./.github/actions/install-dependencies - with: - version: ${{ matrix.deps }} - - - uses: actions/setup-python@v5 - with: - python-version: "${{ matrix.python }}" - allow-prereleases: true - - name: Install packages run: | sudo apt-get update -y -qq - sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev + sudo apt-get install -y -qq libboost-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev libgeos-dev liblz4-dev virtualenv if: ${{ matrix.flavour == 'linux' }} - name: Install packages - run: brew install boost geos + run: brew install boost geos virtualenv shell: bash if: ${{ matrix.flavour == 'macos' }} - - name: Install prerequisites + - name: Setup virtualenv run: | - python -m pip install --upgrade pip - pip install -U pytest pytest-httpserver shapely setuptools requests + virtualenv venv + ./venv/bin/pip install build + + - name: Install package dependencies (develop) + run: | + git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git contrib/libosmium + git clone --quiet --depth 1 https://github.com/mapbox/protozero.git contrib/protozero + git clone --quiet https://github.com/pybind/pybind11.git contrib/pybind11 shell: bash + if: ${{ matrix.deps == 'develop' }} + + - name: Install package dependencies (release) + run: | + ./venv/bin/python -m build --sdist + shell: bash + if: ${{ matrix.deps == 'release' }} + + - uses: actions/setup-python@v5 + with: + python-version: "${{ matrix.python }}" + allow-prereleases: true - name: Build package - run: python setup.py build + run: | + ./venv/bin/python -m build -w shell: bash - name: Run tests run: | - pytest test + WHEEL=`ls dist/osmium*.whl` + echo "Installing $WHEEL" + ./venv/bin/pip install dist/osmium*whl ${WHEEL}[tests] + ./venv/bin/pytest test shell: bash @@ -476,11 +490,16 @@ jobs: env: CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake + - name: Repair wheels + run: | + pip install delvewheel + delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl + - name: 'Upload Artifact' uses: actions/upload-artifact@v4 with: name: pyosmium-win64-dist - path: dist + path: wheelhouse build-windows-free-threaded: runs-on: windows-2022 @@ -502,7 +521,8 @@ jobs: shell: bash - name: Install packages - run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows + run: | + vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows boost-variant:x64-windows boost-iterator:x64-windows lz4:x86-windows shell: bash - name: Set up Python 3.13t @@ -532,11 +552,16 @@ jobs: env: CMAKE_TOOLCHAIN_FILE: C:/vcpkg/scripts/buildsystems/vcpkg.cmake + - name: Repair wheels + run: | + pip install delvewheel + delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin/ --add-path C:/vcpkg/installed/x64-windows/debug/bin dist/*.whl + - name: 'Upload Artifact' uses: actions/upload-artifact@v4 with: name: pyosmium-win64-dist-t - path: dist + path: wheelhouse test-windows: @@ -599,7 +624,7 @@ jobs: test-windows-free-threaded: runs-on: windows-2022 - needs: build-windows + needs: build-windows-free-threaded strategy: fail-fast: false @@ -646,7 +671,8 @@ jobs: if: matrix.python-version == '3.14t' - name: Run tests - run: ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5 + run: | + ./osmium-test/Scripts/pytest test --parallel-threads 5 --iterations 5 shell: bash - name: Check tool availability diff --git a/CMakeLists.txt b/CMakeLists.txt index c37874f4..2f492c34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,74 +1,73 @@ -cmake_minimum_required(VERSION 3.15.0) -project(pyosmium VERSION 4.0.2) +cmake_minimum_required(VERSION 3.15...4.0) +cmake_policy(VERSION 3.15) +project(osmium LANGUAGES CXX) option(WITH_LZ4 "Build with lz4 support for PBF files" ON) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +message(STATUS "Module path: ${CMAKE_MODULE_PATH}") -find_package(Osmium 2.16 REQUIRED COMPONENTS io pbf xml) +set(PYBIND11_FINDPYTHON ON) +find_package(pybind11 CONFIG REQUIRED) + +if(SKBUILD_STATE STREQUAL "sdist") +####################################################################### +# source dist collection +# + +include(FetchContent) + +FetchContent_Declare(libosmium + GIT_REPOSITORY https://github.com/osmcode/libosmium + GIT_TAG 85aa0ec170d99b432d29a372554b27491d28065e # release 2.22.0 + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/contrib/libosmium + SOURCE_SUBDIR cmake + GIT_SUBMODULES_RECURSE OFF) +FetchContent_Declare(protozero + GIT_REPOSITORY https://github.com/mapbox/protozero + GIT_TAG 89a55ad2962cca3adbe8383a4b6d9a8411352ef2 # release 1.8.1 + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/contrib/protozero + SOURCE_SUBDIR cmake + GIT_SUBMODULES_RECURSE OFF) + +FetchContent_MakeAvailable(libosmium protozero) + +else() +####################################################################### +# wheel build +# + +set(OSMIUM_COMPONENTS io pbf xml) if(WITH_LZ4) -find_package(LZ4) - - if(LZ4_FOUND) - message(STATUS "lz4 library found, compiling with it") - add_definitions(-DOSMIUM_WITH_LZ4) - include_directories(SYSTEM ${LZ4_INCLUDE_DIRS}) - list(APPEND OSMIUM_LIBRARIES ${LZ4_LIBRARIES}) - else() - message(WARNING "lz4 library not found, compiling without it") - endif() + find_package(LZ4) + if (LZ4_FOUND) + list(APPEND OSMIUM_COMPONENTS lz4) + message(STATUS "lz4 library found, compiling with it") + else() + message(WARNING "lz4 library not found, compiling without it") + endif() else() - message(STATUS "Building without lz4 support: Set WITH_LZ4=ON to change this") + message(STATUS "Building without lz4 support: Set WITH_LZ4=ON to change this") endif() -include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS} ${PROTOZERO_INCLUDE_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib) +find_package(Osmium 2.16 REQUIRED COMPONENTS ${OSMIUM_COMPONENTS}) +include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS}) if(NOT "${CMAKE_CXX_STANDARD}") set(CMAKE_CXX_STANDARD 17) endif() -set(PYBIND11_CPP_STANDARD -std=c++${CMAKE_CXX_STANDARD}) - message(STATUS "Building in C++${CMAKE_CXX_STANDARD} mode") -find_package(Python COMPONENTS Interpreter Development) - -# Check for abiflags, so we can check for free-threaded later. -execute_process(COMMAND ${Python_EXECUTABLE} -c "import sys; print(sys.abiflags, end='')" - OUTPUT_VARIABLE PYTHON_ABIFLAGS) - -if(PYBIND11_PREFIX) - add_subdirectory(${PYBIND11_PREFIX} contrib/pybind11) -elseif(PYTHON_ABIFLAGS STREQUAL "t") - message(STATUS "Free-threading Python found. Enabling support (needs pybind11 2.13+).") - find_package(pybind11 2.13 REQUIRED) -else() - find_package(pybind11 2.9 REQUIRED) -endif() - -find_package(Boost 1.70 REQUIRED COMPONENTS) +find_package(Boost 1.66 REQUIRED COMPONENTS) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) -function(set_module_output module outdir) - set_target_properties(${module} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${outdir}) - # windows needs a build type variant - foreach(config ${CMAKE_CONFIGURATION_TYPES}) - string(TOUPPER ${config} config) - set_target_properties(${module} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY_${config} - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${outdir}) - endforeach() -endfunction() - # Modules without any Python code and just one source file. foreach(PYMOD geom index io area) pybind11_add_module(${PYMOD} lib/${PYMOD}.cc) - set_module_output(${PYMOD} osmium) target_link_libraries(${PYMOD} PRIVATE ${OSMIUM_LIBRARIES}) + install(TARGETS ${PYMOD} DESTINATION osmium) if(APPLE) set_target_properties(${PYMOD} PROPERTIES CXX_VISIBILITY_PRESET "default") endif() @@ -77,8 +76,8 @@ endforeach() # Modules where additional Python code is in src (C++-part will be private). foreach(PYMOD osm replication) pybind11_add_module(_${PYMOD} lib/${PYMOD}.cc) - set_module_output(_${PYMOD} osmium/${PYMOD}) target_link_libraries(_${PYMOD} PRIVATE ${OSMIUM_LIBRARIES}) + install(TARGETS _${PYMOD} DESTINATION osmium/${PYMOD}) if(APPLE) set_target_properties(_${PYMOD} PROPERTIES CXX_VISIBILITY_PRESET "default") endif() @@ -92,7 +91,7 @@ pybind11_add_module(_osmium lib/simple_writer.cc lib/file_iterator.cc lib/id_tracker.cc) -set_module_output(_osmium osmium) +install(TARGETS _osmium DESTINATION osmium) target_link_libraries(_osmium PRIVATE ${OSMIUM_LIBRARIES}) pybind11_add_module(filter @@ -103,12 +102,17 @@ pybind11_add_module(filter lib/id_filter.cc lib/entity_filter.cc lib/geo_interface_filter.cc) -set_module_output(filter osmium) +install(TARGETS filter DESTINATION osmium) target_link_libraries(filter PRIVATE ${OSMIUM_LIBRARIES}) +configure_file(cmake/version.py.tmpl version.py) +install(FILES ${PROJECT_BINARY_DIR}/version.py DESTINATION osmium) + # workaround for https://github.com/pybind/pybind11/issues/1272 if(APPLE) set_target_properties(_osmium PROPERTIES CXX_VISIBILITY_PRESET "default") set_target_properties(filter PROPERTIES CXX_VISIBILITY_PRESET "default") endif() + +endif() diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index fbfe9e36..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,15 +0,0 @@ -include README.md -include CHANGELOG.md -include LICENSE.TXT -include CMakeLists.txt -include mkdocs.yaml -include cmake/*.cmake -include lib/*.h -include lib/*.cc -include examples/*py -include test/*py -include src/osmium/py.typed -recursive-include src *.pyi -recursive-include docs *md *css -include docs/cookbooks/*.ipynb -include docs/Makefile diff --git a/README.md b/README.md index f5ec8e37..b6c9055a 100644 --- a/README.md +++ b/README.md @@ -42,41 +42,40 @@ pyosmium has the following dependencies: * [libbz2](https://www.sourceware.org/bzip2/) * [Boost](https://www.boost.org/) variant and iterator >= 1.70 * [Python Requests](https://docs.python-requests.org/) - * Python setuptools + * [scikit-build-core](https://scikit-build-core.readthedocs.io) * a C++17-compatible compiler (Clang 13+, GCC 10+ are supported) ### Compiling from Source -Get the latest versions of libosmium, protozero and pybind11 source code. It is -recommended that you put them in a subdirectory `contrib`. +Make sure to install the development packages for expat, libz, libbz2 +and boost. -You can do this by cloning their repositories into that location. +The appropriate versions for Libosmium and Protozero will be downloaded into +the `contrib` directory when building the source package: -Following commands should achieve this: + python3 -m build -s -``` -mkdir contrib -cd contrib -git clone https://github.com/osmcode/libosmium.git -git clone https://github.com/mapbox/protozero.git -git clone https://github.com/pybind/pybind11.git -``` +Alternatively, provide custom locations for these libraries by setting +`Libosmium_ROOT` and `Protozero_ROOT`. -You can also set custom locations with `LIBOSMIUM_PREFIX`, `PROTOZERO_PREFIX` and -`PYBIND11_PREFIX` respectively. +To compile and install the bindings, run -To use a custom boost installation, set `BOOST_PREFIX`. + pip install . -To compile the bindings during development, you can use -[build](https://pypa-build.readthedocs.io/en/stable/). -On Debian/Ubuntu-like systems, install `python3-build`, then -run: +### Compiling for Development - python3 -m build -w +To compile during development, you can use the experimental +[Editable install mode](https://scikit-build-core.readthedocs.io/en/latest/configuration/index.html#editable-installs) +of scikit-build-core: -To compile and install the bindings, run +Create a virtualenv with scikit-build-core and pybind11 preinstalled: - pip install . + virtualenv /tmp/dev-venv + /tmp/dev-venv/bin/pip install scikit-build-core pybind11 + +Now compile pyosmium with: + + /tmp/dev-venv/bin/pip --no-build-isolation --config-settings=editable.rebuild=true -Cbuild-dir=/tmp/build -ve. ## Examples @@ -90,18 +89,18 @@ They are mostly ports of the examples in Libosmium and osmium-contrib. There is a small test suite in the test directory. This provides unit test for the python bindings, it is not meant to be a test suite for Libosmium. -Testing requires `pytest` and `pytest-httpserver`. On Debian/Ubuntu install -the dependencies with: +Testing requires `pytest` and `pytest-httpserver` and optionally +pytest-run-parallel and shapely. Install those into your dev environment: - sudo apt-get install python3-pytest python3-pytest-httpserver + /tmp/dev-venv/bin/pip install --no-build-isolation --config-settings=editable.rebuild=true -Cbuild-dir=build -ve.[tests] -or install them with pip using: +The test suite can be run with: - pip install osmium[tests] + /tmp/dev-venv/bin/pytest test -The test suite can be run with: +To test parallel execution on free-threaded Python, run: - pytest test + /tmp/dev-venv/bin/pytest test --parallel-threads 10 --iterations 100 ## Documentation @@ -145,4 +144,5 @@ Pyosmium is available under the BSD 2-Clause License. See LICENSE.TXT. ## Authors -Sarah Hoffmann (lonvia@denofr.de) +Sarah Hoffmann (lonvia@denofr.de) and otheres. See commit logs for a full +list. diff --git a/cmake/FindOsmium.cmake b/cmake/FindOsmium.cmake index 76a71507..336ec093 100644 --- a/cmake/FindOsmium.cmake +++ b/cmake/FindOsmium.cmake @@ -55,19 +55,10 @@ # #---------------------------------------------------------------------- -# This is the list of directories where we look for osmium includes. -set(_osmium_include_path - ../libosmium - ~/Library/Frameworks - /Library/Frameworks - /opt/local # DarwinPorts - /opt -) - # Look for the header file. find_path(OSMIUM_INCLUDE_DIR osmium/version.hpp PATH_SUFFIXES include - PATHS ${_osmium_include_path} + HINTS $ENV{LIBOSMIUM_PREFIX} ${CMAKE_SOURCE_DIR}/contrib/libosmium ) # Check libosmium version number @@ -290,6 +281,7 @@ find_package_handle_standard_args(Osmium REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS} VERSION_VAR _libosmium_version) unset(OSMIUM_EXTRA_FIND_VARS) +set(OSMIUM_VERSION ${_libosmium_version}) #---------------------------------------------------------------------- # diff --git a/cmake/FindProtozero.cmake b/cmake/FindProtozero.cmake index ad16cabe..a4289603 100644 --- a/cmake/FindProtozero.cmake +++ b/cmake/FindProtozero.cmake @@ -39,7 +39,7 @@ # find include path find_path(PROTOZERO_INCLUDE_DIR protozero/version.hpp PATH_SUFFIXES include - PATHS ${CMAKE_SOURCE_DIR}/../protozero + HINTS $ENV{PROTOZERO_PREFIX} ${CMAKE_SOURCE_DIR}/contrib/protozero ) # Check version number @@ -58,6 +58,6 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Protozero REQUIRED_VARS PROTOZERO_INCLUDE_DIR VERSION_VAR _version) - +SET(PROTOZERO_VERSION ${_version}) #---------------------------------------------------------------------- diff --git a/cmake/version.py.tmpl b/cmake/version.py.tmpl new file mode 100644 index 00000000..9fd40d7e --- /dev/null +++ b/cmake/version.py.tmpl @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# This file is part of pyosmium. (https://osmcode.org/pyosmium/) +# +# Copyright (C) 2025 Sarah Hoffmann and others. +# For a full list of authors see the git log. +""" +Version information. +""" + +# current release (Pip version) +pyosmium_release = '@SKBUILD_PROJECT_VERSION@' + +# libosmium version used to compile this package +libosmium_version = '@OSMIUM_VERSION@' +# protozero version used to compile this package +protozero_version = '@PROTOZERO_VERSION@' +# pybind11 version shipped used to compile this package +pybind11_version = '@pybind11_VERSION@' diff --git a/pyproject.toml b/pyproject.toml index a996ae02..7c217139 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,13 @@ [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core>=0.10", "pybind11>=2.9"] +build-backend = "scikit_build_core.build" [project] name = "osmium" +version = "4.0.2" description = "Python bindings for libosmium, the data processing library for OSM data" -readme = "README.rst" requires-python = ">=3.8" + license = {text = 'BSD-2-Clause'} authors = [ {name = "Sarah Hoffmann", email = "lonvia@denofr.de"} @@ -18,6 +19,7 @@ keywords = ["OSM", "OpenStreetMap", "Osmium"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", + "Topic :: Scientific/Engineering :: GIS", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -29,28 +31,12 @@ classifiers = [ "Programming Language :: Python :: Free Threading :: 2 - Beta", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: C++", + "Typing :: Typed" ] dependencies = [ "requests" ] -dynamic = ["version"] - -[tool.setuptools] -packages = [ - "osmium", - "osmium.osm", - "osmium.replication" - ] -package-dir = {"" = "src"} - - -[project.urls] -Homepage = "https://osmcode.org/pyosmium" -Documentation = "https://docs.osmcode.org/pyosmium/latest/" -Repository = "https://github.com/osmcode/pyosmium" -Issues = "https://github.com/osmcode/pyosmium/issues" - [project.optional-dependencies] tests = [ 'pytest', @@ -70,5 +56,42 @@ docs = [ 'argparse-manpage ' ] -[tool.setuptools.dynamic] -version = {attr = "osmium.version.pyosmium_release"} +[project.scripts] +pyosmium-get-changes = "osmium.tools.pyosmium_get_changes:main" +pyosmium-up-to-date = "osmium.tools.pyosmium_up_to_date:main" + +[tool.scikit-build] +minimum-version = "build-system.requires" +cmake.version = ">=3.15" + +[tool.scikit-build.sdist] +cmake = true +exclude = ['**'] +include = ['/src/**/*.py', + '/src/**/*.pyi', + '/src/osmium/py.typed', + '/test/**/*.py', + '/lib/**/*.cc', + '/lib/**/*.h', + '/cmake/*', + '/docs/*.md', + '/docs/*/*.md', + '/docs/*/*.css', + '/docs/cookbooks/*ipynb', + '/docs/Makefile', + '/examples/*py', + '/mkdocs.yaml', + '/pyproject.toml', + '/README.*', + '/CHANGELOG.md', + '/CMakeLists.txt', + '/LICENSE.TXT', + '/contrib/libosmium/include', + '/contrib/libosmium/CMakeLists.txt', + '/contrib/libosmium/LICENSE', + '/contrib/libosmium/README.md', + '/contrib/protozero/include', + '/contrib/protozero/CMakeLists.txt', + '/contrib/protozero/LICENSE', + '/contrib/protozero/README.md', + ] diff --git a/setup.py b/setup.py deleted file mode 100644 index 0e2e6803..00000000 --- a/setup.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# This file is part of pyosmium. (https://osmcode.org/pyosmium/) -# -# Copyright (C) 2024 Sarah Hoffmann and others. -# For a full list of authors see the git log. -import os -import re -import sys -import platform -import subprocess -import urllib.request -import tarfile -from pathlib import Path - -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist as orig_sdist - -BASEDIR = os.path.split(os.path.abspath(__file__))[0] - -class PyosmiumSDist(orig_sdist): - - contrib = ( - ('libosmium', 'https://github.com/osmcode/libosmium/archive/v{}.tar.gz'), - ('protozero', 'https://github.com/mapbox/protozero/archive/v{}.tar.gz'), - ('pybind11', 'https://github.com/pybind/pybind11/archive/v{}.tar.gz'), - ) - - def make_release_tree(self, base_dir, files): - orig_sdist.make_release_tree(self, base_dir, files) - - # add additional dependecies in the required version - for name, tar_src in self.contrib: - tarball = tar_src.format(versions[name + '_version']) - print("Downloading and adding {} sources from {}".format(name, tarball)) - base = Path("-".join((name, versions[name + '_version']))) - dest = Path(base_dir) / "contrib" / name - with urllib.request.urlopen(tarball) as reader: - with tarfile.open(fileobj=reader, mode='r|gz') as tf: - for member in tf: - fname = Path(member.name) - if not fname.is_absolute(): - fname = fname.relative_to(base) - if member.isdir(): - (dest / fname).mkdir(parents=True, exist_ok=True) - elif member.isfile(): - with tf.extractfile(member) as memberfile: - with (dest / fname).open('wb') as of: - of.write(memberfile.read()) - - - -def get_versions(): - """ Read the version file. - - The file cannot be directly imported because it is not installed - yet. - """ - version_py = os.path.join(BASEDIR, "src/osmium/version.py") - v = {} - with open(version_py) as version_file: - # Execute the code in version.py. - exec(compile(version_file.read(), version_py, 'exec'), v) - - return v - -class CMakeExtension(Extension): - def __init__(self, name, sourcedir=''): - Extension.__init__(self, name, sources=[]) - self.sourcedir = os.path.abspath(sourcedir) - - -class CMakeBuild(build_ext): - def run(self): - try: - out = subprocess.check_output(['cmake', '--version']) - except OSError: - raise RuntimeError("CMake must be installed to build the following extensions: " + - ", ".join(e.name for e in self.extensions)) - - for ext in self.extensions: - self.build_extension(ext) - - def build_extension(self, ext): - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, - '-DPYTHON_EXECUTABLE=' + sys.executable] - - cfg = 'Debug' if self.debug else 'Release' - build_args = ['--config', cfg] - - if platform.system() == "Windows": - cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] - if sys.maxsize > 2**32: - cmake_args += ['-A', 'x64'] - build_args += ['--', '/m'] - else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] - nbr_cpus = os.cpu_count() or 2 # fallback if None is returned - build_args += ['--', f'-j{nbr_cpus}'] - - env = os.environ.copy() - env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), - self.distribution.get_version()) - - if 'LIBOSMIUM_PREFIX' in env: - cmake_args += ['-DOSMIUM_INCLUDE_DIR={}/include'.format(env['LIBOSMIUM_PREFIX'])] - elif os.path.exists(os.path.join(BASEDIR, 'contrib', 'libosmium', 'include', 'osmium', 'version.hpp')): - cmake_args += ['-DOSMIUM_INCLUDE_DIR={}/contrib/libosmium/include'.format(BASEDIR)] - - if 'PROTOZERO_PREFIX' in env: - cmake_args += ['-DPROTOZERO_INCLUDE_DIR={}/include'.format(env['PROTOZERO_PREFIX'])] - elif os.path.exists(os.path.join(BASEDIR, 'contrib', 'protozero', 'include', 'protozero', 'version.hpp')): - cmake_args += ['-DPROTOZERO_INCLUDE_DIR={}/contrib/protozero/include'.format(BASEDIR)] - - if 'PYBIND11_PREFIX' in env: - cmake_args += ['-DPYBIND11_PREFIX={}'.format(env['PYBIND11_PREFIX'])] - elif os.path.exists(os.path.join(BASEDIR, 'contrib', 'pybind11')): - cmake_args += ['-DPYBIND11_PREFIX={}/contrib/pybind11'.format(BASEDIR)] - - if 'BOOST_PREFIX' in env: - cmake_args += ['-DBOOST_ROOT={}'.format(env['BOOST_PREFIX'])] - - if 'CMAKE_CXX_STANDARD' in env: - cmake_args += ['-DCMAKE_CXX_STANDARD={}'.format(env['CMAKE_CXX_STANDARD'])] - - cmake_args += [f"-DWITH_LZ4={env.get('CMAKE_WITH_LZ4', 'ON')}"] - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) - subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) - -versions = get_versions() - -setup( - scripts=['tools/pyosmium-get-changes', 'tools/pyosmium-up-to-date'], - ext_modules=[CMakeExtension('cmake_example')], - cmdclass=dict(build_ext=CMakeBuild, sdist=PyosmiumSDist), - zip_safe=False, -) diff --git a/src/osmium/tools/__init__.py b/src/osmium/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/pyosmium-get-changes b/src/osmium/tools/pyosmium_get_changes.py old mode 100755 new mode 100644 similarity index 89% rename from tools/pyosmium-get-changes rename to src/osmium/tools/pyosmium_get_changes.py index 02af7486..d747c556 --- a/tools/pyosmium-get-changes +++ b/src/osmium/tools/pyosmium_get_changes.py @@ -1,4 +1,3 @@ -#!/usr/bin/python """ Fetch diffs from an OSM planet server. @@ -24,6 +23,9 @@ However, it can read cookies from a Netscape-style cookie jar file, send these cookies to the server and will save received cookies to the jar file. """ +import sys +import logging +from textwrap import dedent as msgfmt from argparse import ArgumentParser, RawDescriptionHelpFormatter, ArgumentTypeError import datetime as dt @@ -33,16 +35,11 @@ from osmium.replication import newest_change_from_file from osmium.replication.utils import get_replication_header from osmium.version import pyosmium_release -from osmium import SimpleHandler, SimpleWriter - - -import re -import sys -import logging -from textwrap import dedent as msgfmt +from osmium import SimpleWriter log = logging.getLogger() + class ReplicationStart(object): """ Represents the point where changeset download should begin. """ @@ -78,7 +75,8 @@ def from_date(datestr): date = dt.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") date = date.replace(tzinfo=dt.timezone.utc) except ValueError: - raise ArgumentTypeError("Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).") + raise ArgumentTypeError( + "Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).") return ReplicationStart(date=date) @@ -106,6 +104,7 @@ def from_osm_file(fname, ignore_headers): return ReplicationStart(seq_id=seq, date=ts, src=url) + def write_end_sequence(fname, seqid): """Either writes out the sequence file or prints the sequence id to stdout. """ @@ -115,6 +114,7 @@ def write_end_sequence(fname, seqid): with open(fname, 'w') as fd: fd.write(str(seqid)) + def get_arg_parser(from_main=False): parser = ArgumentParser(prog='pyosmium-get-changes', description=__doc__, @@ -146,10 +146,10 @@ def get_arg_parser(from_main=False): group.add_argument('-O', '--start-osm-data', dest='start_file', metavar='OSMFILE', help='start at the date of the newest OSM object in the file') parser.add_argument('-f', '--sequence-file', dest='seq_file', - help='Sequence file. If the file exists, then updates ' - 'will start after the id given in the file. At the ' - 'end of the process, the last sequence ID contained ' - 'in the diff is written.') + help='Sequence file. If the file exists, then updates ' + 'will start after the id given in the file. At the ' + 'end of the process, the last sequence ID contained ' + 'in the diff is written.') parser.add_argument('--ignore-osmosis-headers', dest='ignore_headers', action='store_true', help='When determining the start from an OSM file, ' @@ -164,7 +164,7 @@ def get_arg_parser(from_main=False): return parser -def main(args): +def pyosmium_get_changes(args): logging.basicConfig(stream=sys.stderr, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') @@ -194,17 +194,16 @@ def main(args): if options.server_url is not None and options.start.source is not None: if options.server_url != options.start.source: - log.error(msgfmt(""" - You asked to use server URL: - %s - but the referenced OSM file points to replication server: - %s - If you really mean to overwrite the URL, use --ignore-osmosis-headers.""" - % (options.server_url, options.start.source))) + log.error(msgfmt(f""" + You asked to use server URL: + {options.server_url} + but the referenced OSM file points to replication server: + {options.start.source} + If you really mean to overwrite the URL, use --ignore-osmosis-headers.""")) return 2 url = options.server_url \ - or options.start.source \ - or 'https://planet.osm.org/replication/minute/' + or options.start.source \ + or 'https://planet.osm.org/replication/minute/' logging.info("Using replication server at %s" % url) with rserv.ReplicationServer(url, diff_type=options.server_diff_type) as svr: @@ -248,5 +247,9 @@ def main(args): return 0 -if __name__ == '__main__': - exit(main(sys.argv[1:])) +def main(): + logging.basicConfig(stream=sys.stderr, + format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + + return pyosmium_get_changes(sys.argv[1:]) diff --git a/tools/pyosmium-up-to-date b/src/osmium/tools/pyosmium_up_to_date.py old mode 100755 new mode 100644 similarity index 92% rename from tools/pyosmium-up-to-date rename to src/osmium/tools/pyosmium_up_to_date.py index 29476e98..74aaf5bf --- a/tools/pyosmium-up-to-date +++ b/src/osmium/tools/pyosmium_up_to_date.py @@ -1,4 +1,3 @@ -#!/usr/bin/python """ Update an OSM file with changes from a OSM replication server. @@ -30,8 +29,6 @@ However, it can read cookies from a Netscape-style cookie jar file, send these cookies to the server and will save received cookies to the jar file. """ - -import re import sys import traceback import logging @@ -49,6 +46,7 @@ log = logging.getLogger() + def update_from_osm_server(ts, options): """Update the OSM file using the official OSM servers at https://planet.osm.org/replication. This strategy will attempt @@ -121,11 +119,11 @@ def update_from_custom_server(url, seq, ts, options): ofname = outfile try: - extra_headers = { 'generator' : 'pyosmium-up-to-date/' + pyosmium_release } + extra_headers = {'generator': 'pyosmium-up-to-date/' + pyosmium_release} outseqs = svr.apply_diffs_to_file(infile, ofname, startseq, - max_size=options.outsize*1024, - extra_headers=extra_headers, - outformat=options.outformat) + max_size=options.outsize*1024, + extra_headers=extra_headers, + outformat=options.outformat) if outseqs is None: log.info("No new updates found.") @@ -157,13 +155,12 @@ def compute_start_point(options): if options.server_url is not None: if url is not None and url != options.server_url: - log.error(msgfmt(""" + log.error(msgfmt(f""" You asked to use server URL: - %s + {options.server_url} but the referenced OSM file points to replication server: - %s - If you really mean to overwrite the URL, use --ignore-osmosis-headers.""" - % (options.server_url, url))) + {url} + If you really mean to overwrite the URL, use --ignore-osmosis-headers.""")) exit(2) url = options.server_url @@ -180,6 +177,7 @@ def compute_start_point(options): return url, seq, ts + def get_arg_parser(from_main=False): parser = ArgumentParser(prog='pyosmium-up-to-date', @@ -218,7 +216,7 @@ def get_arg_parser(from_main=False): help="Apply update even if the input data is really old.") parser.add_argument('--cookie', dest='cookie', help='Netscape-style cookie jar file to read cookies from and where ' - 'received cookies will be written to.') + 'received cookies will be written to.') parser.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60, help='Set timeout for file downloads.') parser.add_argument('--version', action='version', @@ -226,14 +224,8 @@ def get_arg_parser(from_main=False): return parser -def open_with_cookie(url): - return opener.open(url) - -if __name__ == '__main__': - logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S') +def pyosmium_up_to_date(args): options = get_arg_parser(from_main=True).parse_args() log.setLevel(max(3 - options.loglevel, 0) * 10) @@ -241,15 +233,22 @@ def open_with_cookie(url): url, seq, ts = compute_start_point(options) except RuntimeError as e: log.error(str(e)) - exit(2) + return 2 try: if url is None: - ret = update_from_osm_server(ts, options) - else: - ret = update_from_custom_server(url, seq, ts, options) - except: + return update_from_osm_server(ts, options) + + return update_from_custom_server(url, seq, ts, options) + except Exception: traceback.print_exc() - exit(254) - exit(ret) + return 254 + + +def main(): + logging.basicConfig(stream=sys.stderr, + format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + + return pyosmium_up_to_date(sys.argv[1:]) diff --git a/src/osmium/version.py b/src/osmium/version.py deleted file mode 100644 index b42e7d59..00000000 --- a/src/osmium/version.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# This file is part of pyosmium. (https://osmcode.org/pyosmium/) -# -# Copyright (C) 2025 Sarah Hoffmann and others. -# For a full list of authors see the git log. -""" -Version information. -""" - -# the major version -pyosmium_major = '4.0' -# current release (Pip version) -pyosmium_release = '4.0.2' - -# libosmium version shipped with the Pip release -libosmium_version = '2.20.0' -# protozero version shipped with the Pip release -protozero_version = '1.8.1' -# pybind11 version shipped with the Pip release -pybind11_version = '3.0.0' diff --git a/src/osmium/version.pyi b/src/osmium/version.pyi new file mode 100644 index 00000000..8e0fee7a --- /dev/null +++ b/src/osmium/version.pyi @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# This file is part of pyosmium. (https://osmcode.org/pyosmium/) +# +# Copyright (C) 2025 Sarah Hoffmann and others. +# For a full list of authors see the git log. +""" +Typing information for generated version module. +""" + +pyosmium_release: str +libosmium_version: str +protozero_version: str +pybind11_version: str diff --git a/test/conftest.py b/test/conftest.py index b194df54..b1420465 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,31 +5,11 @@ # Copyright (C) 2025 Sarah Hoffmann and others. # For a full list of authors see the git log. from pathlib import Path -import sys -import sysconfig import uuid from textwrap import dedent import pytest - - -SRC_DIR = (Path(__file__) / '..' / '..').resolve() - - -BUILD_DIR = "build/lib.{}-{}.{}".format(sysconfig.get_platform(), - sys.version_info[0], sys.version_info[1]) - -if not (SRC_DIR / BUILD_DIR).exists(): - BUILD_DIR = "build/lib.{}-{}{}".format(sysconfig.get_platform(), - sys.implementation.cache_tag, - getattr(sys, 'abiflags', '')) - -if (SRC_DIR / BUILD_DIR).exists(): - sys.path.insert(0, str(SRC_DIR)) - sys.path.insert(0, str(SRC_DIR / BUILD_DIR)) - - -import osmium # noqa +import osmium @pytest.fixture diff --git a/test/test_pyosmium_get_changes.py b/test/test_pyosmium_get_changes.py index a9578eda..2998121c 100644 --- a/test/test_pyosmium_get_changes.py +++ b/test/test_pyosmium_get_changes.py @@ -6,13 +6,12 @@ # For a full list of authors see the git log. """ Tests for the pyosmium-get-changes script. """ -from pathlib import Path from textwrap import dedent import uuid -import pytest import osmium.replication.server import osmium +from osmium.tools.pyosmium_get_changes import pyosmium_get_changes from helpers import IDCollector @@ -24,16 +23,8 @@ class TestPyosmiumGetChanges: - @pytest.fixture(autouse=True) - def setup(self): - self.script = dict() - - filename = Path(__file__, "..", "..", "tools", "pyosmium-get-changes").resolve() - with filename.open("rb") as f: - exec(compile(f.read(), str(filename), 'exec'), self.script) - def main(self, httpserver, *args): - return self.script['main'](['--server', httpserver.url_for('')] + list(args)) + return pyosmium_get_changes(['--server', httpserver.url_for('')] + list(args)) def test_init_id(self, capsys, httpserver): assert 0 == self.main(httpserver, '-I', '453')