From 7a9c63f2df9be053f62c3f1698097a9bffff7afd Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Fri, 10 Jun 2022 16:53:05 -0700 Subject: [PATCH 01/11] Ubuntu & macOS matrix --- .github/actions/build-and-test/action.yml | 56 --- .github/actions/build/action.yml | 44 ++ .github/actions/install/action.yml | 26 + .../actions/macos-prerequisites/action.yml | 34 ++ .github/actions/test/action.yml | 27 + .../actions/ubuntu-prerequisites/action.yml | 51 +- .github/actions/win-postgres/action.yml | 8 + .github/workflows/ci.yml | 461 +++++++++--------- .github/workflows/test-install.yml | 108 ++-- .gitignore | 1 + 10 files changed, 438 insertions(+), 378 deletions(-) delete mode 100644 .github/actions/build-and-test/action.yml create mode 100644 .github/actions/build/action.yml create mode 100644 .github/actions/install/action.yml create mode 100644 .github/actions/macos-prerequisites/action.yml create mode 100644 .github/actions/test/action.yml diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml deleted file mode 100644 index e7e901732..000000000 --- a/.github/actions/build-and-test/action.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: 'Build and test osm2pgsq' - -inputs: - test-wrapper: - description: 'Wrapper to use around tests' - required: false - default: pg_virtualenv - -runs: - using: "composite" - - steps: - - name: create build directory - run: mkdir build - shell: bash - - - name: configure - run: | - CMAKE_OPTIONS="-LA -DBUILD_TESTS=ON" - if [ -z "${LUA_VERSION}" ]; then - CMAKE_OPTIONS="$CMAKE_OPTIONS -DWITH_LUA=OFF" - else - CMAKE_OPTIONS="$CMAKE_OPTIONS -DWITH_LUAJIT=${LUAJIT_OPTION}" - fi - if [ -n "$USE_PROJ_LIB" ]; then - CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_PROJ_LIB=$USE_PROJ_LIB" - fi - if [ -n "$CPP_VERSION" ]; then - CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_CXX_STANDARD=$CPP_VERSION" - fi - if [ -n "$BUILD_TYPE" ]; then - CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_BUILD_TYPE=$BUILD_TYPE" - fi - cmake $CMAKE_OPTIONS .. - shell: bash - working-directory: build - env: - CXXFLAGS: -pedantic -Wextra -Werror - - - name: build - run: make -j3 all man - shell: bash - working-directory: build - - - name: test - run: | - if [ "$WRAPPER" = "pg_virtualenv" ]; then - pg_virtualenv -v $POSTGRESQL_VERSION ctest --output-on-failure - else - $WRAPPER ctest --output-on-failure - fi - shell: bash - working-directory: build - env: - WRAPPER: ${{ inputs.test-wrapper }} - diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml new file mode 100644 index 000000000..42cd6e152 --- /dev/null +++ b/.github/actions/build/action.yml @@ -0,0 +1,44 @@ +--- +name: Build osm2pgsql + +runs: + using: composite + + steps: + - name: create build directory + run: mkdir build + shell: bash + + - name: configure + run: | + export CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" + export CXXFLAGS=${CXXFLAGS:--pedantic -Werror -Wextra} + + if [ -z "${LUA_VERSION:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUA=OFF" + else + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUAJIT=${LUAJIT_OPTION:-OFF}" + fi + if [ -n "${USE_PROJ_LIB:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DUSE_PROJ_LIB=${USE_PROJ_LIB}" + fi + if [ -n "${CPP_VERSION:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_CXX_STANDARD=${CPP_VERSION}" + fi + if [ -n "${BUILD_TESTS:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_TESTS=${BUILD_TESTS}" + fi + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" + fi + cmake \ + ${CMAKE_OPTIONS} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/usr/local} \ + .. + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build + + - name: build + run: make -j4 all man + shell: bash + working-directory: build diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml new file mode 100644 index 000000000..55f7d5652 --- /dev/null +++ b/.github/actions/install/action.yml @@ -0,0 +1,26 @@ +--- +name: Install osm2pgsql & check installation + +runs: + using: composite + + steps: + - name: install + run: sudo make install + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build + + - name: check + run: | + export INSTALL_PREFIX=${INSTALL_PREFIX:-/usr/local} + + test -d ${INSTALL_PREFIX}/bin + test -e ${INSTALL_PREFIX}/bin/osm2pgsql + test -e ${INSTALL_PREFIX}/bin/osm2pgsql-replication + test -d ${INSTALL_PREFIX}/share/man/man1 + test -f ${INSTALL_PREFIX}/share/man/man1/osm2pgsql.1 + test -f ${INSTALL_PREFIX}/share/man/man1/osm2pgsql-replication.1 + test -d ${INSTALL_PREFIX}/share/osm2pgsql + test -f ${INSTALL_PREFIX}/share/osm2pgsql/default.style + test -f ${INSTALL_PREFIX}/share/osm2pgsql/empty.style + shell: bash --noprofile --norc -euxo pipefail {0} diff --git a/.github/actions/macos-prerequisites/action.yml b/.github/actions/macos-prerequisites/action.yml new file mode 100644 index 000000000..0c2fabfb6 --- /dev/null +++ b/.github/actions/macos-prerequisites/action.yml @@ -0,0 +1,34 @@ +--- +name: Install Prerequisites on macOS + +runs: + using: composite + + steps: + - name: Install software + run: | + brew install boost lua pandoc postgis + pip3 install behave psycopg2 + shell: bash --noprofile --norc -euxo pipefail {0} + + - name: Start database + run: | + mkdir -p ${PGDATA} + pg_ctl init --pgdata=${PGDATA} + echo "fsync = off" >> ${PGDATA}/postgresql.conf + echo "full_page_writes = off" >> ${PGDATA}/postgresql.conf + echo "shared_buffers = 1GB" >> ${PGDATA}/postgresql.conf + echo "synchronous_commit = off" >> ${PGDATA}/postgresql.conf + echo "unix_socket_directories = ''" >> ${PGDATA}/postgresql.conf + pg_ctl start --pgdata=${PGDATA} + shell: bash --noprofile --norc -euxo pipefail {0} + + - name: Setup database + run: | + pg_isready --timeout=30 + mkdir -m 777 /tmp/tst + psql -c "CREATE TABLESPACE tablespacetest LOCATION '/tmp/tst'" + psql -c "CREATE EXTENSION hstore" + psql -c "CREATE EXTENSION postgis" + psql -c "SELECT PostGIS_Full_Version()" + shell: bash --noprofile --norc -euxo pipefail {0} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 000000000..c19404bc3 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,27 @@ +--- +name: Test osm2pgsql + +inputs: + wrapper: + description: Wrapper to use around tests + required: false + +runs: + using: composite + + steps: + - name: test + env: + WRAPPER: ${{ inputs.wrapper }} + run: | + CTEST_CMD="ctest --output-on-failure" + + if [ "${WRAPPER}" == "pg_virtualenv" ]; then + ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} + elif [ -n "${WRAPPER}" ]; then + ${WRAPPER} ${CTEST_CMD} + else + ${CTEST_CMD} + fi + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build diff --git a/.github/actions/ubuntu-prerequisites/action.yml b/.github/actions/ubuntu-prerequisites/action.yml index f86ad2f22..2e9c97975 100644 --- a/.github/actions/ubuntu-prerequisites/action.yml +++ b/.github/actions/ubuntu-prerequisites/action.yml @@ -1,19 +1,20 @@ -name: 'Install Prerequisites on Ubuntu' +--- +name: Install Prerequisites on Ubuntu runs: - using: "composite" + using: composite steps: - name: Remove preinstalled postgresql run: | - sudo apt-get remove -yq postgresql* + sudo apt-get purge -qq postgresql* sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo apt-get update -qq - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} - name: Install software run: | - sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + sudo apt-get install -qq --no-install-suggests --no-install-recommends \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ @@ -24,32 +25,50 @@ runs: postgresql-${POSTGRESQL_VERSION} \ postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} \ postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION}-scripts \ - postgresql-client postgresql-contrib-${POSTGRESQL_VERSION} \ + postgresql-client-${POSTGRESQL_VERSION} \ + postgresql-contrib-${POSTGRESQL_VERSION} \ python3-psycopg2 \ + python3-pyosmium \ python3-setuptools \ zlib1g-dev pip3 install behave - if [ "$CC" = clang-8 ]; then sudo apt-get install -yq --no-install-suggests --no-install-recommends clang-8; fi - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} + + - name: Install compiler and set CC/CXX + run: | + if [ -z "${CC:-}" ]; then + export CC=gcc + echo "CC=${CC}" >> $GITHUB_ENV + fi + if [ "${CC:0:3}" == "gcc" ]; then + export CXX=${CC/gcc/g++} + echo "CXX=${CXX}" >> $GITHUB_ENV + sudo apt-get install -qq --no-install-suggests --no-install-recommends ${CC} ${CXX} + elif [ "${CC:0:5}" == "clang" ]; then + export CXX=${CC/clang/clang++} + echo "CXX=${CXX}" >> $GITHUB_ENV + sudo apt-get install -qq --no-install-suggests --no-install-recommends ${CC} + fi + shell: bash --noprofile --norc -euxo pipefail {0} - name: Install Lua run: | - if [ -n "${LUA_VERSION}" ]; then - sudo apt-get install -yq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION} + if [ -n "${LUA_VERSION:-}" ]; then + sudo apt-get install -qq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION} fi - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} - name: Install LuaJIT run: | - if [ "${LUAJIT_OPTION}" = "ON" ]; then - sudo apt-get install -yq --no-install-suggests --no-install-recommends libluajit-5.1-dev + if [ "${LUAJIT_OPTION:-}" = "ON" ]; then + sudo apt-get install -qq --no-install-suggests --no-install-recommends libluajit-5.1-dev fi - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} - name: Adapt postgresql configuration run: | echo 'fsync = off' | sudo tee /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf - echo 'synchronous_commit = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf echo 'full_page_writes = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf echo 'shared_buffers = 1GB' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf - shell: bash + echo 'synchronous_commit = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf + shell: bash --noprofile --norc -euxo pipefail {0} diff --git a/.github/actions/win-postgres/action.yml b/.github/actions/win-postgres/action.yml index 6bae12470..225d7054b 100644 --- a/.github/actions/win-postgres/action.yml +++ b/.github/actions/win-postgres/action.yml @@ -2,6 +2,7 @@ name: Set up postgresql on Windows runs: using: composite + steps: - name: Download postgis run: | @@ -13,6 +14,13 @@ runs: echo "Root: $PGROOT, Bin: $PGBIN" cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" shell: bash + - name: Configure PostgreSQL + run: | + echo "fsync = off" >> "$PGROOT\data\postgresql.conf" + echo "full_page_writes = off" >> "$PGROOT\data\postgresql.conf" + echo "shared_buffers = 1GB" >> "$PGROOT\data\postgresql.conf" + echo "synchronous_commit = off" >> "$PGROOT\data\postgresql.conf" + shell: bash - name: Start PostgreSQL on Windows run: | $pgService = Get-Service -Name postgresql* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78b6da663..53064dc5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,262 +1,232 @@ +--- name: CI -on: [ push, pull_request ] +on: [push, pull_request] -jobs: - macos: - runs-on: macos-latest - env: - LUA_VERSION: 5.4 - steps: - - uses: actions/checkout@v2 - - - name: Install prerequisites - run: | - brew install lua boost postgis pandoc - pip3 install psycopg2 behave - pg_ctl -D /usr/local/var/postgres start - shell: bash - - - name: Setup database - run: | - psql -d postgres -c 'CREATE EXTENSION postgis' - psql -d postgres -c 'CREATE EXTENSION hstore' - psql -d postgres -c 'SELECT PostGIS_Full_Version()' - mkdir -m 700 $GITHUB_WORKSPACE/tablespacetest - psql -d postgres -c "CREATE TABLESPACE tablespacetest LOCATION '$GITHUB_WORKSPACE/tablespacetest'" - shell: bash - - - uses: ./.github/actions/build-and-test - with: - test-wrapper: '' - env: - PGHOST: /tmp - - - ubuntu18-pg95-gcc7-jit: - runs-on: ubuntu-18.04 - - env: - CC: gcc-7 - CXX: g++-7 - LUA_VERSION: 5.3 - LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 9.5 - POSTGIS_VERSION: 2.4 - BUILD_TYPE: Release - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu18-pg96-clang8-jit: - runs-on: ubuntu-18.04 - - env: - CC: clang-8 - CXX: clang++-8 - LUA_VERSION: 5.3 - LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 9.6 - POSTGIS_VERSION: 2.4 - BUILD_TYPE: Release - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu18-pg10-gcc9: - runs-on: ubuntu-18.04 - - env: - CC: gcc-9 - CXX: g++-9 - LUA_VERSION: 5.3 - LUAJIT_OPTION: OFF - POSTGRESQL_VERSION: 10 - POSTGIS_VERSION: 3 - CPP_VERSION: 17 - BUILD_TYPE: Debug - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - - ubuntu18-pg11-clang9: - runs-on: ubuntu-18.04 - - env: - CC: clang-9 - CXX: clang++-9 - LUA_VERSION: 5.3 - LUAJIT_OPTION: OFF - POSTGRESQL_VERSION: 11 - POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 - BUILD_TYPE: Debug - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu20-pg12-gcc10-jit: - runs-on: ubuntu-20.04 - - env: - CC: gcc-10 - CXX: g++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 12 - POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 - BUILD_TYPE: Debug - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - - ubuntu20-pg13-clang10-jit: - runs-on: ubuntu-20.04 +defaults: + run: + shell: bash +jobs: + macOS: env: - CC: clang-10 - CXX: clang++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 13 - POSTGIS_VERSION: 3 - CPP_VERSION: 17 - BUILD_TYPE: Debug + BUILD_TESTS: ON + BUILD_TYPE: ${{ matrix.build_type || 'Debug' }} + LUA_VERSION: ${{ matrix.lua_version || '' }} + LUAJIT_OPTION: ${{ matrix.luajit_option && 'ON' || 'OFF' }} + PGDATA: /tmp/postgresql/data + PGDATABASE: postgres + PGHOST: localhost - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test + name: macos${{ matrix.image_tag }} - ubuntu20-pg13-clang10-proj6: - runs-on: ubuntu-20.04 + runs-on: macos-${{ matrix.image_tag }} - env: - CC: clang-10 - CXX: clang++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: OFF - POSTGRESQL_VERSION: 13 - POSTGIS_VERSION: 3 - CPP_VERSION: 17 - USE_PROJ_LIB: 6 - BUILD_TYPE: Debug + strategy: + fail-fast: false + matrix: + image_tag: + - 10.15 + - 11 + - 12 + lua_version: + - 5.4 steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu20-pg13-clang10-noproj: - runs-on: ubuntu-20.04 + - uses: actions/checkout@v3 + - uses: ./.github/actions/macos-prerequisites + - uses: ./.github/actions/build + - uses: ./.github/actions/test + Ubuntu: env: - CC: clang-10 - CXX: clang++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: OFF - POSTGRESQL_VERSION: 13 - POSTGIS_VERSION: 3 - CPP_VERSION: 17 - USE_PROJ_LIB: off - BUILD_TYPE: Debug + BUILD_TESTS: ON + BUILD_TYPE: ${{ matrix.build_type || 'Debug' }} + CC: ${{ matrix.cc || 'gcc' }} + CPP_VERSION: ${{ matrix.cpp_version || '' }} + DEBIAN_FRONTEND: noninteractive + LUA_VERSION: ${{ matrix.lua_version || '' }} + LUAJIT_OPTION: ${{ matrix.luajit_option && 'ON' || 'OFF' }} + POSTGIS_VERSION: ${{ matrix.postgis_version || '3' }} + POSTGRESQL_VERSION: ${{ matrix.postgresql_version || '14' }} + USE_PROJ_LIB: ${{ matrix.use_proj_lib || '' }} + + name: >- + ubuntu${{ matrix.image_tag }} + pg${{ matrix.postgresql_version || '13' }} + ${{ matrix.cc || 'gcc' }} + ${{ matrix.build_type == 'Release' && 'release' || '' }} + ${{ matrix.lua_version && format('lua{0}', matrix.lua_version) || 'nolua' }} + ${{ matrix.luajit_option && 'jit' || '' }} + ${{ matrix.cpp_version && format('cpp{0}', matrix.cpp_version) || '' }} + ${{ matrix.use_proj_lib && (matrix.use_proj_lib == 'off' && 'noproj' || format('proj{0}', matrix.use_proj_lib)) || '' }} + + runs-on: ubuntu-${{ matrix.image_tag }} - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu20-pg13-clang10-cpp17: - runs-on: ubuntu-20.04 - - env: - CC: clang-10 - CXX: clang++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: OFF - POSTGRESQL_VERSION: 14 - POSTGIS_VERSION: 3 - CPP_VERSION: 17 - USE_PROJ_LIB: 6 - BUILD_TYPE: Debug + strategy: + fail-fast: false + matrix: + image_tag: + # - 18.04 # Clang 6.0.0 | GCC 7.5.0 + # - 20.04 # Clang 10.0.0 | GCC 9.4.0 + - 22.04 # Clang 14.0.0 | GCC 11.2.0 + cc: + - clang + - gcc + lua_version: + - 5.4 + - false + luajit_option: + - false + - true + postgis_version: + - 3 + postgresql_version: + - 14 + exclude: + - lua_version: false + luajit_option: true + include: + # ubuntu18-pg95-gcc7-jit + - build_type: Release + cc: gcc-7 + image_tag: 18.04 + lua_version: 5.3 + luajit_option: true + postgis_version: 2.5 + postgresql_version: 9.5 + # ubuntu18-pg96-clang8-jit + - build_type: Release + cc: clang-8 + image_tag: 18.04 + lua_version: 5.3 + luajit_option: true + postgis_version: 2.5 + postgresql_version: 9.6 + # ubuntu18-pg10-gcc9 + - build_type: Debug + cc: gcc-9 + cpp_version: 17 + image_tag: 18.04 + lua_version: 5.3 + luajit_option: false + postgis_version: 3 + postgresql_version: 10 + # ubuntu18-pg11-clang9 + - build_type: Debug + cc: clang-9 + cpp_version: 17 + image_tag: 18.04 + lua_version: false + luajit_option: false + postgis_version: 2.5 + postgresql_version: 11 + # ubuntu20-pg12-gcc10-jit + - build_type: Debug + cc: gcc-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: true + postgis_version: 2.5 + postgresql_version: 12 + # ubuntu20-pg13-clang10-jit + - build_type: Debug + cc: clang-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: true + postgis_version: 3 + postgresql_version: 13 + # ubuntu20-pg13-clang10-proj6 + - build_type: Debug + cc: clang-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: false + postgis_version: 3 + postgresql_version: 13 + use_proj_lib: 6 + # ubuntu20-pg13-clang10-noproj + - build_type: Debug + cc: clang-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: false + postgis_version: 3 + postgresql_version: 13 + use_proj_lib: off + # ubuntu20-pg14-clang10-cpp17 + - build_type: Debug + cc: clang-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: false + postgis_version: 3 + postgresql_version: 14 + use_proj_lib: 6 + # ubuntu20-pg13-gcc10-release + - build_type: Release + cc: gcc-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: 5.3 + luajit_option: true + postgis_version: 2.5 + postgresql_version: 13 + # ubuntu20-pg13-gcc10-release-nolua + - build_type: Release + cc: gcc-10 + cpp_version: 17 + image_tag: 20.04 + lua_version: false + luajit_option: false + postgis_version: 2.5 + postgresql_version: 13 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - ubuntu20-pg13-gcc10-release: - runs-on: ubuntu-20.04 + - uses: ./.github/actions/build + - uses: ./.github/actions/test + with: + wrapper: pg_virtualenv + Windows: env: - CC: gcc-10 - CXX: g++-10 - LUA_VERSION: 5.3 - LUAJIT_OPTION: ON - POSTGRESQL_VERSION: 13 - POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 - BUILD_TYPE: Release - - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test + GETOPT_INCLUDE_DIR: ${{ github.workspace }}/../wingetopt/src + GETOPT_LIBRARY: ${{ github.workspace }}/../wingetopt/build/Release/wingetopt.lib + VCPKG_DEFAULT_BINARY_CACHE: C:\vcpkg_binary_cache - ubuntu20-pg13-gcc10-release-nolua: - runs-on: ubuntu-20.04 + name: windows${{ matrix.image_tag }} - env: - CC: gcc-10 - CXX: g++-10 - POSTGRESQL_VERSION: 13 - POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 - BUILD_TYPE: Release + runs-on: windows-${{ matrix.image_tag }} - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build-and-test - - windows: strategy: fail-fast: false matrix: - os: [windows-2019, windows-2022] - runs-on: ${{ matrix.os }} - env: - GETOPT_INCLUDE_DIR: ${{ github.workspace }}/../wingetopt/src - GETOPT_LIBRARY: ${{ github.workspace }}/../wingetopt/build/Release/wingetopt.lib - VCPKG_DEFAULT_BINARY_CACHE: C:/vcpkg_binary_cache + image_tag: [2019, 2022] + steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 + - uses: actions/checkout@v3 + - uses: actions/cache@v3 with: - path: | - C:/vcpkg_binary_cache - key: vcpkg-binary-cache-${{ matrix.os }} - - uses: actions/cache@v2 + path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }} + key: vcpkg-binary-cache-windows-${{ matrix.image_tag }} + - uses: actions/cache@v3 with: path: | C:/postgis.zip key: postgis-cache - name: Prepare cache - run: if [ ! -d C:/vcpkg_binary_cache ]; then mkdir C:/vcpkg_binary_cache; fi - shell: bash + run: | + if [ ! -d "${VCPKG_DEFAULT_BINARY_CACHE}" ]; then + mkdir "${VCPKG_DEFAULT_BINARY_CACHE}" + fi - uses: ./.github/actions/win-postgres - uses: ./.github/actions/win-install - uses: ./.github/actions/win-getopt @@ -268,40 +238,45 @@ jobs: mkdir c:/artifact/ mkdir c:/artifact/osm2pgsql-bin cp -r Release/* ../README.md ../COPYING ../*.style ../scripts ../flex-config c:/artifact/osm2pgsql-bin/ - shell: bash working-directory: build - if: matrix.os == 'windows-2022' + if: matrix.image_tag == '2022' - name: 'Upload Artifact' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: osm2pgsql-win64 path: c:/artifact - if: matrix.os == 'windows-2022' + if: matrix.image_tag == '2022' windows-package: - needs: windows - runs-on: windows-2022 env: + OSMFILE: monaco-latest.osm.bz2 OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.bz2 + + name: windows2022 (Install & import OSM data) + + needs: Windows + + runs-on: windows-2022 + steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 + - uses: actions/checkout@v3 + - uses: actions/cache@v3 with: path: | C:/postgis.zip key: postgis-cache - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: osm2pgsql-win64 - uses: ./.github/actions/win-postgres - name: Set up database run: | & $env:PGBIN\createdb osm - & $env:PGBIN\psql -d osm -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis;" + & $env:PGBIN\psql -d osm -c "CREATE EXTENSION hstore;" + & $env:PGBIN\psql -d osm -c "CREATE EXTENSION postgis;" shell: pwsh - name: Get test data - run: (new-object net.webclient).DownloadFile($env:OSMURL, "testfile.osm.bz2") - - name: Exceute osm2pgsql - run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.bz2 - shell: bash - + run: (new-object net.webclient).DownloadFile($env:OSMURL, $env:OSMFILE) + shell: pwsh + - name: Execute osm2pgsql + run: ./osm2pgsql-bin/osm2pgsql --slim -d osm ${OSMFILE} diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index b4ce52750..ae148ecdd 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -1,79 +1,56 @@ +--- name: Test install -on: [ push, pull_request ] +on: [push, pull_request] + +defaults: + run: + shell: bash jobs: ubuntu-test-install: - runs-on: ubuntu-20.04 - env: - LUA_VERSION: 5.3 - POSTGRESQL_VERSION: 12 - POSTGIS_VERSION: 3 BUILD_TYPE: Release - CXXFLAGS: -pedantic -Wextra -Werror - PREFIX: /usr/local - OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + INSTALL_PREFIX: /usr + LUA_VERSION: 5.3 OSMFILE: monaco-latest.osm.pbf + OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + POSTGIS_VERSION: 3 + POSTGRESQL_VERSION: 12 + + name: Build, install & import OSM data (Ubuntu ${{ matrix.image_tag }}) + + runs-on: ubuntu-${{ matrix.image_tag }} + + strategy: + fail-fast: false + matrix: + image_tag: + - 18.04 # Clang 6.0.0 | GCC 7.5.0 + - 20.04 # Clang 10.0.0 | GCC 9.4.0 + - 22.04 # Clang 14.0.0 | GCC 11.2.0 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Show installed PostgreSQL packages - run: apt-cache search postgresql | sort - - name: Install prerequisites - run: | - sudo apt-get purge -yq postgresql* - sudo apt-get install -yq --no-install-suggests --no-install-recommends \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbz2-dev \ - libexpat1-dev \ - liblua${LUA_VERSION}-dev \ - libluajit-5.1-dev \ - libpq-dev \ - libproj-dev \ - lua${LUA_VERSION} \ - pandoc \ - postgresql-${POSTGRESQL_VERSION} \ - postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} \ - postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION}-scripts \ - postgresql-client-${POSTGRESQL_VERSION} \ - python3-pyosmium \ - python3-psycopg2 \ - zlib1g-dev - - name: Run CMake - run: | - mkdir build - cd build - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE - - name: Build osm2pgsql - working-directory: build - run: make -j3 all man - - name: Install osm2pgsql - working-directory: build - run: sudo make install - - name: Check osm2pgsql install - run: | - test -d $PREFIX/bin - test -e $PREFIX/bin/osm2pgsql - test -e $PREFIX/bin/osm2pgsql-replication - test -d $PREFIX/share/man/man1 - test -f $PREFIX/share/man/man1/osm2pgsql.1 - test -f $PREFIX/share/man/man1/osm2pgsql-replication.1 - test -d $PREFIX/share/osm2pgsql - test -f $PREFIX/share/osm2pgsql/default.style - test -f $PREFIX/share/osm2pgsql/empty.style + run: dpkg --list | grep -i -e postgis -e postgresql | sort + - name: Disregard stringop-overread warnings (Ubuntu 22.04/GCC 11) + run: echo "CXXFLAGS=-pedantic -Werror -Wextra -Wno-stringop-overread" >> $GITHUB_ENV + if: matrix.image_tag == 22.04 + - uses: ./.github/actions/ubuntu-prerequisites + - uses: ./.github/actions/build + - uses: ./.github/actions/install - name: Set up test databases run: | sudo systemctl start postgresql sudo -u postgres createuser runner sudo -u postgres createdb -O runner o2ptest - sudo -u postgres psql o2ptest -c "CREATE EXTENSION postgis;" sudo -u postgres psql o2ptest -c "CREATE EXTENSION hstore;" + sudo -u postgres psql o2ptest -c "CREATE EXTENSION postgis;" + working-directory: /tmp - name: Remove repository # Remove contents of workspace to be sure the install runs independently - working-directory: / - run: rm -fr "${{github.workspace}}"/* + run: rm -frv "${GITHUB_WORKSPACE}"/* - name: Show man pages run: | man -P cat osm2pgsql @@ -82,13 +59,18 @@ jobs: run: wget --quiet $OSMURL working-directory: /tmp - name: Test run of osm2pgsql - run: $PREFIX/bin/osm2pgsql -d o2ptest --slim $OSMFILE + run: | + ${INSTALL_PREFIX}/bin/osm2pgsql \ + -v -d o2ptest --slim $OSMFILE working-directory: /tmp - name: Test run osm2pgsql-replication run: | - $PREFIX/bin/osm2pgsql-replication init -v -d o2ptest - $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest - $PREFIX/bin/osm2pgsql-replication update -v -d o2ptest --once --max-diff-size=1 - $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest --json + ${INSTALL_PREFIX}/bin/osm2pgsql-replication init \ + -v -d o2ptest + ${INSTALL_PREFIX}/bin/osm2pgsql-replication status \ + -v -d o2ptest + ${INSTALL_PREFIX}/bin/osm2pgsql-replication update \ + -v -d o2ptest --once --max-diff-size=1 + ${INSTALL_PREFIX}/bin/osm2pgsql-replication status \ + -v -d o2ptest --json working-directory: /tmp - diff --git a/.gitignore b/.gitignore index b8e1ea0f4..eee5413b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build +!.github/actions/build docs/html docs/latex From bd96a1f296b6f6a233651c25d8d24e26f3fcb05d Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Mon, 13 Jun 2022 22:23:54 -0700 Subject: [PATCH 02/11] Share build & test actions with Windows --- .github/actions/build/action.yml | 23 ++++++-- .github/actions/install/action.yml | 4 +- .../actions/macos-prerequisites/action.yml | 2 +- .github/actions/test/action.yml | 4 ++ .github/actions/win-build/action.yml | 10 ---- .github/actions/win-cmake/action.yml | 13 ---- .github/actions/win-getopt/action.yml | 18 ------ .github/actions/win-install/action.yml | 11 ---- .github/actions/win-test/action.yml | 10 ---- .../action.yml | 9 ++- .../actions/windows-prerequisites/action.yml | 59 +++++++++++++++++++ .github/workflows/ci.yml | 27 ++++++--- .github/workflows/test-install.yml | 3 +- 13 files changed, 111 insertions(+), 82 deletions(-) delete mode 100644 .github/actions/win-build/action.yml delete mode 100644 .github/actions/win-cmake/action.yml delete mode 100644 .github/actions/win-getopt/action.yml delete mode 100644 .github/actions/win-install/action.yml delete mode 100644 .github/actions/win-test/action.yml rename .github/actions/{win-postgres => windows-postgres}/action.yml (91%) create mode 100644 .github/actions/windows-prerequisites/action.yml diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 42cd6e152..cc35d30ee 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -11,8 +11,7 @@ runs: - name: configure run: | - export CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" - export CXXFLAGS=${CXXFLAGS:--pedantic -Werror -Wextra} + CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" if [ -z "${LUA_VERSION:-}" ]; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUA=OFF" @@ -31,14 +30,28 @@ runs: if [ -n "${BUILD_TYPE:-}" ]; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" fi + if [ -n "${INSTALL_PREFIX:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + fi cmake \ ${CMAKE_OPTIONS} \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX:-/usr/local} \ .. shell: bash --noprofile --norc -euxo pipefail {0} working-directory: build - name: build - run: make -j4 all man - shell: bash + run: | + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS:-}" + + if [ -n "${BUILD_PARALLEL_LEVEL:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --parallel ${BUILD_PARALLEL_LEVEL}" + fi + if [ -n "${BUILD_TARGET:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --target ${BUILD_TARGET}" + fi + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --config ${BUILD_TYPE}" + fi + cmake --build . ${CMAKE_BUILD_OPTIONS} + shell: bash --noprofile --norc -euxo pipefail {0} working-directory: build diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 55f7d5652..04af5101f 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -7,12 +7,12 @@ runs: steps: - name: install run: sudo make install - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash working-directory: build - name: check run: | - export INSTALL_PREFIX=${INSTALL_PREFIX:-/usr/local} + INSTALL_PREFIX=${INSTALL_PREFIX:-/usr/local} test -d ${INSTALL_PREFIX}/bin test -e ${INSTALL_PREFIX}/bin/osm2pgsql diff --git a/.github/actions/macos-prerequisites/action.yml b/.github/actions/macos-prerequisites/action.yml index 0c2fabfb6..507d3593e 100644 --- a/.github/actions/macos-prerequisites/action.yml +++ b/.github/actions/macos-prerequisites/action.yml @@ -9,7 +9,7 @@ runs: run: | brew install boost lua pandoc postgis pip3 install behave psycopg2 - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash - name: Start database run: | diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index c19404bc3..d2d8174bf 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -16,6 +16,10 @@ runs: run: | CTEST_CMD="ctest --output-on-failure" + if [ -n "${BUILD_TYPE:-}" ]; then + CTEST_CMD="${CTEST_CMD} --build-config ${BUILD_TYPE}" + fi + if [ "${WRAPPER}" == "pg_virtualenv" ]; then ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} elif [ -n "${WRAPPER}" ]; then diff --git a/.github/actions/win-build/action.yml b/.github/actions/win-build/action.yml deleted file mode 100644 index 4ae3cd9af..000000000 --- a/.github/actions/win-build/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Windows build - -runs: - using: composite - steps: - - name: Build - run: cmake --build . --config Release --verbose - shell: bash - working-directory: build - diff --git a/.github/actions/win-cmake/action.yml b/.github/actions/win-cmake/action.yml deleted file mode 100644 index 1845abe5c..000000000 --- a/.github/actions/win-cmake/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Windows CMake - -runs: - using: composite - steps: - - name: Create build directory - run: mkdir build - shell: bash - - name: Configure - run: cmake -LA .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DBUILD_TESTS=ON -DGETOPT_INCLUDE_DIR=$GETOPT_INCLUDE_DIR -DGETOPT_LIBRARY=$GETOPT_LIBRARY -DBoost_USE_STATIC_LIBS=ON - shell: bash - working-directory: build - diff --git a/.github/actions/win-getopt/action.yml b/.github/actions/win-getopt/action.yml deleted file mode 100644 index 8f303db64..000000000 --- a/.github/actions/win-getopt/action.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Windows getopt - -runs: - using: composite - steps: - - name: Install wingetopt - env: - WINGETOPT_VER: v0.95 - run: | - git clone --quiet --depth 1 https://github.com/alex85k/wingetopt -b $WINGETOPT_VER ../wingetopt - mkdir ../wingetopt/build - shell: bash - - name: Build wingetopt - run: | - cmake -LA .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake - cmake --build . --config Release --verbose - working-directory: ../wingetopt/build - shell: bash diff --git a/.github/actions/win-install/action.yml b/.github/actions/win-install/action.yml deleted file mode 100644 index 9a97b6b93..000000000 --- a/.github/actions/win-install/action.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: Windows install - -runs: - using: composite - steps: - - name: Install packages - run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows - shell: bash - - name: Install psycopg2 and beahve - run: python -m pip install psycopg2 behave - shell: bash diff --git a/.github/actions/win-test/action.yml b/.github/actions/win-test/action.yml deleted file mode 100644 index 7e24c6394..000000000 --- a/.github/actions/win-test/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Windows test - -runs: - using: composite - steps: - - name: Test - run: ctest --output-on-failure -C Release - shell: bash - working-directory: build - diff --git a/.github/actions/win-postgres/action.yml b/.github/actions/windows-postgres/action.yml similarity index 91% rename from .github/actions/win-postgres/action.yml rename to .github/actions/windows-postgres/action.yml index 225d7054b..c025ec173 100644 --- a/.github/actions/win-postgres/action.yml +++ b/.github/actions/windows-postgres/action.yml @@ -1,3 +1,4 @@ +--- name: Set up postgresql on Windows runs: @@ -9,24 +10,28 @@ runs: if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile("https://osm2pgsql.org/ci/winbuild/postgis-bundle-pg14-3.2.0x64.zip", "c:\postgis.zip")} 7z x c:\postgis.zip -oc:\postgis_archive shell: pwsh + - name: Install postgis run: | echo "Root: $PGROOT, Bin: $PGBIN" cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} + - name: Configure PostgreSQL run: | echo "fsync = off" >> "$PGROOT\data\postgresql.conf" echo "full_page_writes = off" >> "$PGROOT\data\postgresql.conf" echo "shared_buffers = 1GB" >> "$PGROOT\data\postgresql.conf" echo "synchronous_commit = off" >> "$PGROOT\data\postgresql.conf" - shell: bash + shell: bash --noprofile --norc -euxo pipefail {0} + - name: Start PostgreSQL on Windows run: | $pgService = Get-Service -Name postgresql* Set-Service -InputObject $pgService -Status running -StartupType automatic Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru shell: pwsh + - name: Create test tablespace run: | mkdir c:\tablespace diff --git a/.github/actions/windows-prerequisites/action.yml b/.github/actions/windows-prerequisites/action.yml new file mode 100644 index 000000000..20bb4fdce --- /dev/null +++ b/.github/actions/windows-prerequisites/action.yml @@ -0,0 +1,59 @@ +--- +name: Install Prerequisites on Windows + +runs: + using: composite + + steps: + - name: Install software + run: | + vcpkg install \ + boost-filesystem:x64-windows \ + boost-property-tree:x64-windows \ + boost-system:x64-windows \ + bzip2:x64-windows \ + expat:x64-windows \ + libpq:x64-windows \ + lua:x64-windows \ + proj4:x64-windows \ + zlib:x64-windows + python -m pip install behave psycopg2 + shell: bash + + - name: Clone wingetopt + env: + WINGETOPT_VER: v0.95 + run: | + git clone \ + --branch ${WINGETOPT_VER} \ + --depth 1 \ + --quiet \ + https://github.com/alex85k/wingetopt + mkdir wingetopt/build + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: .. + + - name: Configure wingetopt + run: | + cmake ${CMAKE_OPTIONS:-} .. + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: ../wingetopt/build + + - name: Build wingetopt + run: | + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS:-}" + + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --config ${BUILD_TYPE}" + fi + cmake --build . ${CMAKE_BUILD_OPTIONS} + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: ../wingetopt/build + + - name: Add GETOPT_INCLUDE_DIR & GETOPT_LIBRARY to CMAKE_OPTIONS + run: | + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DGETOPT_INCLUDE_DIR=${PWD}/src -DGETOPT_LIBRARY=${PWD}/build/Release/wingetopt.lib" + + echo "CMAKE_OPTIONS=${CMAKE_OPTIONS}" >> $GITHUB_ENV + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: ../wingetopt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53064dc5f..4a54a1821 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,11 @@ defaults: jobs: macOS: env: + BUILD_PARALLEL_LEVEL: 4 + BUILD_TARGET: all man BUILD_TESTS: ON BUILD_TYPE: ${{ matrix.build_type || 'Debug' }} + CXXFLAGS: -pedantic -Werror -Wextra LUA_VERSION: ${{ matrix.lua_version || '' }} LUAJIT_OPTION: ${{ matrix.luajit_option && 'ON' || 'OFF' }} PGDATA: /tmp/postgresql/data @@ -40,10 +43,13 @@ jobs: Ubuntu: env: + BUILD_PARALLEL_LEVEL: 4 + BUILD_TARGET: all man BUILD_TESTS: ON BUILD_TYPE: ${{ matrix.build_type || 'Debug' }} CC: ${{ matrix.cc || 'gcc' }} CPP_VERSION: ${{ matrix.cpp_version || '' }} + CXXFLAGS: -pedantic -Werror -Wextra DEBIAN_FRONTEND: noninteractive LUA_VERSION: ${{ matrix.lua_version || '' }} LUAJIT_OPTION: ${{ matrix.luajit_option && 'ON' || 'OFF' }} @@ -198,8 +204,13 @@ jobs: Windows: env: - GETOPT_INCLUDE_DIR: ${{ github.workspace }}/../wingetopt/src - GETOPT_LIBRARY: ${{ github.workspace }}/../wingetopt/build/Release/wingetopt.lib + BUILD_TESTS: ON + BUILD_TYPE: Release + CMAKE_OPTIONS: >- + -DBoost_USE_STATIC_LIBS=ON + -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake + -LA + LUA_VERSION: ON VCPKG_DEFAULT_BINARY_CACHE: C:\vcpkg_binary_cache name: windows${{ matrix.image_tag }} @@ -227,12 +238,10 @@ jobs: if [ ! -d "${VCPKG_DEFAULT_BINARY_CACHE}" ]; then mkdir "${VCPKG_DEFAULT_BINARY_CACHE}" fi - - uses: ./.github/actions/win-postgres - - uses: ./.github/actions/win-install - - uses: ./.github/actions/win-getopt - - uses: ./.github/actions/win-cmake - - uses: ./.github/actions/win-build - - uses: ./.github/actions/win-test + - uses: ./.github/actions/windows-postgres + - uses: ./.github/actions/windows-prerequisites + - uses: ./.github/actions/build + - uses: ./.github/actions/test - name: Package osm2pgsql run: | mkdir c:/artifact/ @@ -268,7 +277,7 @@ jobs: - uses: actions/download-artifact@v3 with: name: osm2pgsql-win64 - - uses: ./.github/actions/win-postgres + - uses: ./.github/actions/windows-postgres - name: Set up database run: | & $env:PGBIN\createdb osm diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index ae148ecdd..0ad454a34 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -11,6 +11,7 @@ jobs: ubuntu-test-install: env: BUILD_TYPE: Release + CXXFLAGS: -pedantic -Werror -Wextra INSTALL_PREFIX: /usr LUA_VERSION: 5.3 OSMFILE: monaco-latest.osm.pbf @@ -35,7 +36,7 @@ jobs: - name: Show installed PostgreSQL packages run: dpkg --list | grep -i -e postgis -e postgresql | sort - name: Disregard stringop-overread warnings (Ubuntu 22.04/GCC 11) - run: echo "CXXFLAGS=-pedantic -Werror -Wextra -Wno-stringop-overread" >> $GITHUB_ENV + run: echo "CXXFLAGS=${CXXFLAGS} -Wno-stringop-overread" >> $GITHUB_ENV if: matrix.image_tag == 22.04 - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build From 907ae4b2e7bdd7b1dfd417f197ebc21eed835c54 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 08:52:02 -0700 Subject: [PATCH 03/11] Revert `.github/workflows/test-install.yml` --- .github/workflows/test-install.yml | 109 +++++++++++++++++------------ 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 0ad454a34..b4ce52750 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -1,57 +1,79 @@ ---- name: Test install -on: [push, pull_request] - -defaults: - run: - shell: bash +on: [ push, pull_request ] jobs: ubuntu-test-install: + runs-on: ubuntu-20.04 + env: - BUILD_TYPE: Release - CXXFLAGS: -pedantic -Werror -Wextra - INSTALL_PREFIX: /usr LUA_VERSION: 5.3 - OSMFILE: monaco-latest.osm.pbf - OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf - POSTGIS_VERSION: 3 POSTGRESQL_VERSION: 12 - - name: Build, install & import OSM data (Ubuntu ${{ matrix.image_tag }}) - - runs-on: ubuntu-${{ matrix.image_tag }} - - strategy: - fail-fast: false - matrix: - image_tag: - - 18.04 # Clang 6.0.0 | GCC 7.5.0 - - 20.04 # Clang 10.0.0 | GCC 9.4.0 - - 22.04 # Clang 14.0.0 | GCC 11.2.0 + POSTGIS_VERSION: 3 + BUILD_TYPE: Release + CXXFLAGS: -pedantic -Wextra -Werror + PREFIX: /usr/local + OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + OSMFILE: monaco-latest.osm.pbf steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Show installed PostgreSQL packages - run: dpkg --list | grep -i -e postgis -e postgresql | sort - - name: Disregard stringop-overread warnings (Ubuntu 22.04/GCC 11) - run: echo "CXXFLAGS=${CXXFLAGS} -Wno-stringop-overread" >> $GITHUB_ENV - if: matrix.image_tag == 22.04 - - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build - - uses: ./.github/actions/install + run: apt-cache search postgresql | sort + - name: Install prerequisites + run: | + sudo apt-get purge -yq postgresql* + sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbz2-dev \ + libexpat1-dev \ + liblua${LUA_VERSION}-dev \ + libluajit-5.1-dev \ + libpq-dev \ + libproj-dev \ + lua${LUA_VERSION} \ + pandoc \ + postgresql-${POSTGRESQL_VERSION} \ + postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} \ + postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION}-scripts \ + postgresql-client-${POSTGRESQL_VERSION} \ + python3-pyosmium \ + python3-psycopg2 \ + zlib1g-dev + - name: Run CMake + run: | + mkdir build + cd build + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE + - name: Build osm2pgsql + working-directory: build + run: make -j3 all man + - name: Install osm2pgsql + working-directory: build + run: sudo make install + - name: Check osm2pgsql install + run: | + test -d $PREFIX/bin + test -e $PREFIX/bin/osm2pgsql + test -e $PREFIX/bin/osm2pgsql-replication + test -d $PREFIX/share/man/man1 + test -f $PREFIX/share/man/man1/osm2pgsql.1 + test -f $PREFIX/share/man/man1/osm2pgsql-replication.1 + test -d $PREFIX/share/osm2pgsql + test -f $PREFIX/share/osm2pgsql/default.style + test -f $PREFIX/share/osm2pgsql/empty.style - name: Set up test databases run: | sudo systemctl start postgresql sudo -u postgres createuser runner sudo -u postgres createdb -O runner o2ptest - sudo -u postgres psql o2ptest -c "CREATE EXTENSION hstore;" sudo -u postgres psql o2ptest -c "CREATE EXTENSION postgis;" - working-directory: /tmp + sudo -u postgres psql o2ptest -c "CREATE EXTENSION hstore;" - name: Remove repository # Remove contents of workspace to be sure the install runs independently - run: rm -frv "${GITHUB_WORKSPACE}"/* + working-directory: / + run: rm -fr "${{github.workspace}}"/* - name: Show man pages run: | man -P cat osm2pgsql @@ -60,18 +82,13 @@ jobs: run: wget --quiet $OSMURL working-directory: /tmp - name: Test run of osm2pgsql - run: | - ${INSTALL_PREFIX}/bin/osm2pgsql \ - -v -d o2ptest --slim $OSMFILE + run: $PREFIX/bin/osm2pgsql -d o2ptest --slim $OSMFILE working-directory: /tmp - name: Test run osm2pgsql-replication run: | - ${INSTALL_PREFIX}/bin/osm2pgsql-replication init \ - -v -d o2ptest - ${INSTALL_PREFIX}/bin/osm2pgsql-replication status \ - -v -d o2ptest - ${INSTALL_PREFIX}/bin/osm2pgsql-replication update \ - -v -d o2ptest --once --max-diff-size=1 - ${INSTALL_PREFIX}/bin/osm2pgsql-replication status \ - -v -d o2ptest --json + $PREFIX/bin/osm2pgsql-replication init -v -d o2ptest + $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest + $PREFIX/bin/osm2pgsql-replication update -v -d o2ptest --once --max-diff-size=1 + $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest --json working-directory: /tmp + From 30dfa1cc3ec098a324a4c8aed2817e4cfab36c53 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 08:55:05 -0700 Subject: [PATCH 04/11] Revert `Windows` changes --- .github/actions/win-build/action.yml | 10 +++ .github/actions/win-cmake/action.yml | 13 +++ .github/actions/win-getopt/action.yml | 18 ++++ .github/actions/win-install/action.yml | 11 +++ .../action.yml | 15 +--- .github/actions/win-test/action.yml | 10 +++ .../actions/windows-prerequisites/action.yml | 59 ------------- .github/workflows/ci.yml | 87 ++++++++----------- 8 files changed, 99 insertions(+), 124 deletions(-) create mode 100644 .github/actions/win-build/action.yml create mode 100644 .github/actions/win-cmake/action.yml create mode 100644 .github/actions/win-getopt/action.yml create mode 100644 .github/actions/win-install/action.yml rename .github/actions/{windows-postgres => win-postgres}/action.yml (68%) create mode 100644 .github/actions/win-test/action.yml delete mode 100644 .github/actions/windows-prerequisites/action.yml diff --git a/.github/actions/win-build/action.yml b/.github/actions/win-build/action.yml new file mode 100644 index 000000000..4ae3cd9af --- /dev/null +++ b/.github/actions/win-build/action.yml @@ -0,0 +1,10 @@ +name: Windows build + +runs: + using: composite + steps: + - name: Build + run: cmake --build . --config Release --verbose + shell: bash + working-directory: build + diff --git a/.github/actions/win-cmake/action.yml b/.github/actions/win-cmake/action.yml new file mode 100644 index 000000000..1845abe5c --- /dev/null +++ b/.github/actions/win-cmake/action.yml @@ -0,0 +1,13 @@ +name: Windows CMake + +runs: + using: composite + steps: + - name: Create build directory + run: mkdir build + shell: bash + - name: Configure + run: cmake -LA .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DBUILD_TESTS=ON -DGETOPT_INCLUDE_DIR=$GETOPT_INCLUDE_DIR -DGETOPT_LIBRARY=$GETOPT_LIBRARY -DBoost_USE_STATIC_LIBS=ON + shell: bash + working-directory: build + diff --git a/.github/actions/win-getopt/action.yml b/.github/actions/win-getopt/action.yml new file mode 100644 index 000000000..8f303db64 --- /dev/null +++ b/.github/actions/win-getopt/action.yml @@ -0,0 +1,18 @@ +name: Windows getopt + +runs: + using: composite + steps: + - name: Install wingetopt + env: + WINGETOPT_VER: v0.95 + run: | + git clone --quiet --depth 1 https://github.com/alex85k/wingetopt -b $WINGETOPT_VER ../wingetopt + mkdir ../wingetopt/build + shell: bash + - name: Build wingetopt + run: | + cmake -LA .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake + cmake --build . --config Release --verbose + working-directory: ../wingetopt/build + shell: bash diff --git a/.github/actions/win-install/action.yml b/.github/actions/win-install/action.yml new file mode 100644 index 000000000..9a97b6b93 --- /dev/null +++ b/.github/actions/win-install/action.yml @@ -0,0 +1,11 @@ +name: Windows install + +runs: + using: composite + steps: + - name: Install packages + run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows + shell: bash + - name: Install psycopg2 and beahve + run: python -m pip install psycopg2 behave + shell: bash diff --git a/.github/actions/windows-postgres/action.yml b/.github/actions/win-postgres/action.yml similarity index 68% rename from .github/actions/windows-postgres/action.yml rename to .github/actions/win-postgres/action.yml index c025ec173..6bae12470 100644 --- a/.github/actions/windows-postgres/action.yml +++ b/.github/actions/win-postgres/action.yml @@ -1,37 +1,24 @@ ---- name: Set up postgresql on Windows runs: using: composite - steps: - name: Download postgis run: | if (!(Test-Path "C:\postgis.zip")){(new-object net.webclient).DownloadFile("https://osm2pgsql.org/ci/winbuild/postgis-bundle-pg14-3.2.0x64.zip", "c:\postgis.zip")} 7z x c:\postgis.zip -oc:\postgis_archive shell: pwsh - - name: Install postgis run: | echo "Root: $PGROOT, Bin: $PGBIN" cp -r c:/postgis_archive/postgis-bundle-*/* "$PGROOT" - shell: bash --noprofile --norc -euxo pipefail {0} - - - name: Configure PostgreSQL - run: | - echo "fsync = off" >> "$PGROOT\data\postgresql.conf" - echo "full_page_writes = off" >> "$PGROOT\data\postgresql.conf" - echo "shared_buffers = 1GB" >> "$PGROOT\data\postgresql.conf" - echo "synchronous_commit = off" >> "$PGROOT\data\postgresql.conf" - shell: bash --noprofile --norc -euxo pipefail {0} - + shell: bash - name: Start PostgreSQL on Windows run: | $pgService = Get-Service -Name postgresql* Set-Service -InputObject $pgService -Status running -StartupType automatic Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru shell: pwsh - - name: Create test tablespace run: | mkdir c:\tablespace diff --git a/.github/actions/win-test/action.yml b/.github/actions/win-test/action.yml new file mode 100644 index 000000000..7e24c6394 --- /dev/null +++ b/.github/actions/win-test/action.yml @@ -0,0 +1,10 @@ +name: Windows test + +runs: + using: composite + steps: + - name: Test + run: ctest --output-on-failure -C Release + shell: bash + working-directory: build + diff --git a/.github/actions/windows-prerequisites/action.yml b/.github/actions/windows-prerequisites/action.yml deleted file mode 100644 index 20bb4fdce..000000000 --- a/.github/actions/windows-prerequisites/action.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: Install Prerequisites on Windows - -runs: - using: composite - - steps: - - name: Install software - run: | - vcpkg install \ - boost-filesystem:x64-windows \ - boost-property-tree:x64-windows \ - boost-system:x64-windows \ - bzip2:x64-windows \ - expat:x64-windows \ - libpq:x64-windows \ - lua:x64-windows \ - proj4:x64-windows \ - zlib:x64-windows - python -m pip install behave psycopg2 - shell: bash - - - name: Clone wingetopt - env: - WINGETOPT_VER: v0.95 - run: | - git clone \ - --branch ${WINGETOPT_VER} \ - --depth 1 \ - --quiet \ - https://github.com/alex85k/wingetopt - mkdir wingetopt/build - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: .. - - - name: Configure wingetopt - run: | - cmake ${CMAKE_OPTIONS:-} .. - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: ../wingetopt/build - - - name: Build wingetopt - run: | - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS:-}" - - if [ -n "${BUILD_TYPE:-}" ]; then - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --config ${BUILD_TYPE}" - fi - cmake --build . ${CMAKE_BUILD_OPTIONS} - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: ../wingetopt/build - - - name: Add GETOPT_INCLUDE_DIR & GETOPT_LIBRARY to CMAKE_OPTIONS - run: | - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DGETOPT_INCLUDE_DIR=${PWD}/src -DGETOPT_LIBRARY=${PWD}/build/Release/wingetopt.lib" - - echo "CMAKE_OPTIONS=${CMAKE_OPTIONS}" >> $GITHUB_ENV - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: ../wingetopt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a54a1821..f3ed3cf05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,90 +202,75 @@ jobs: with: wrapper: pg_virtualenv - Windows: - env: - BUILD_TESTS: ON - BUILD_TYPE: Release - CMAKE_OPTIONS: >- - -DBoost_USE_STATIC_LIBS=ON - -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake - -LA - LUA_VERSION: ON - VCPKG_DEFAULT_BINARY_CACHE: C:\vcpkg_binary_cache - - name: windows${{ matrix.image_tag }} - - runs-on: windows-${{ matrix.image_tag }} - + windows: strategy: fail-fast: false matrix: - image_tag: [2019, 2022] - + os: [windows-2019, windows-2022] + runs-on: ${{ matrix.os }} + env: + GETOPT_INCLUDE_DIR: ${{ github.workspace }}/../wingetopt/src + GETOPT_LIBRARY: ${{ github.workspace }}/../wingetopt/build/Release/wingetopt.lib + VCPKG_DEFAULT_BINARY_CACHE: C:/vcpkg_binary_cache steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: actions/checkout@v2 + - uses: actions/cache@v2 with: - path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }} - key: vcpkg-binary-cache-windows-${{ matrix.image_tag }} - - uses: actions/cache@v3 + path: | + C:/vcpkg_binary_cache + key: vcpkg-binary-cache-${{ matrix.os }} + - uses: actions/cache@v2 with: path: | C:/postgis.zip key: postgis-cache - name: Prepare cache - run: | - if [ ! -d "${VCPKG_DEFAULT_BINARY_CACHE}" ]; then - mkdir "${VCPKG_DEFAULT_BINARY_CACHE}" - fi - - uses: ./.github/actions/windows-postgres - - uses: ./.github/actions/windows-prerequisites - - uses: ./.github/actions/build - - uses: ./.github/actions/test + run: if [ ! -d C:/vcpkg_binary_cache ]; then mkdir C:/vcpkg_binary_cache; fi + shell: bash + - uses: ./.github/actions/win-postgres + - uses: ./.github/actions/win-install + - uses: ./.github/actions/win-getopt + - uses: ./.github/actions/win-cmake + - uses: ./.github/actions/win-build + - uses: ./.github/actions/win-test - name: Package osm2pgsql run: | mkdir c:/artifact/ mkdir c:/artifact/osm2pgsql-bin cp -r Release/* ../README.md ../COPYING ../*.style ../scripts ../flex-config c:/artifact/osm2pgsql-bin/ + shell: bash working-directory: build - if: matrix.image_tag == '2022' + if: matrix.os == 'windows-2022' - name: 'Upload Artifact' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v2 with: name: osm2pgsql-win64 path: c:/artifact - if: matrix.image_tag == '2022' + if: matrix.os == 'windows-2022' windows-package: + needs: windows + runs-on: windows-2022 env: - OSMFILE: monaco-latest.osm.bz2 OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.bz2 - - name: windows2022 (Install & import OSM data) - - needs: Windows - - runs-on: windows-2022 - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: actions/checkout@v2 + - uses: actions/cache@v2 with: path: | C:/postgis.zip key: postgis-cache - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v2 with: name: osm2pgsql-win64 - - uses: ./.github/actions/windows-postgres + - uses: ./.github/actions/win-postgres - name: Set up database run: | & $env:PGBIN\createdb osm - & $env:PGBIN\psql -d osm -c "CREATE EXTENSION hstore;" - & $env:PGBIN\psql -d osm -c "CREATE EXTENSION postgis;" + & $env:PGBIN\psql -d osm -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis;" shell: pwsh - name: Get test data - run: (new-object net.webclient).DownloadFile($env:OSMURL, $env:OSMFILE) - shell: pwsh - - name: Execute osm2pgsql - run: ./osm2pgsql-bin/osm2pgsql --slim -d osm ${OSMFILE} + run: (new-object net.webclient).DownloadFile($env:OSMURL, "testfile.osm.bz2") + - name: Exceute osm2pgsql + run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.bz2 + shell: bash From 554d874c5e17ea32a18a31bb2aa00b8bc355b8bc Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 08:58:01 -0700 Subject: [PATCH 05/11] Re-combine `build` and `test` --- .../{build => build-and-test}/action.yml | 27 +++++++++++++++- .github/actions/test/action.yml | 31 ------------------- .github/workflows/ci.yml | 8 ++--- 3 files changed, 29 insertions(+), 37 deletions(-) rename .github/actions/{build => build-and-test}/action.yml (72%) delete mode 100644 .github/actions/test/action.yml diff --git a/.github/actions/build/action.yml b/.github/actions/build-and-test/action.yml similarity index 72% rename from .github/actions/build/action.yml rename to .github/actions/build-and-test/action.yml index cc35d30ee..1205c5b46 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -1,5 +1,10 @@ --- -name: Build osm2pgsql +name: Build and test osm2pgsql + +inputs: + test-wrapper: + description: Wrapper to use around tests + required: false runs: using: composite @@ -55,3 +60,23 @@ runs: cmake --build . ${CMAKE_BUILD_OPTIONS} shell: bash --noprofile --norc -euxo pipefail {0} working-directory: build + + - name: test + env: + WRAPPER: ${{ inputs.test-wrapper }} + run: | + CTEST_CMD="ctest --output-on-failure" + + if [ -n "${BUILD_TYPE:-}" ]; then + CTEST_CMD="${CTEST_CMD} --build-config ${BUILD_TYPE}" + fi + + if [ "${WRAPPER}" == "pg_virtualenv" ]; then + ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} + elif [ -n "${WRAPPER}" ]; then + ${WRAPPER} ${CTEST_CMD} + else + ${CTEST_CMD} + fi + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index d2d8174bf..000000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: Test osm2pgsql - -inputs: - wrapper: - description: Wrapper to use around tests - required: false - -runs: - using: composite - - steps: - - name: test - env: - WRAPPER: ${{ inputs.wrapper }} - run: | - CTEST_CMD="ctest --output-on-failure" - - if [ -n "${BUILD_TYPE:-}" ]; then - CTEST_CMD="${CTEST_CMD} --build-config ${BUILD_TYPE}" - fi - - if [ "${WRAPPER}" == "pg_virtualenv" ]; then - ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} - elif [ -n "${WRAPPER}" ]; then - ${WRAPPER} ${CTEST_CMD} - else - ${CTEST_CMD} - fi - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ed3cf05..39dd43cfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/macos-prerequisites - - uses: ./.github/actions/build - - uses: ./.github/actions/test + - uses: ./.github/actions/build-and-test Ubuntu: env: @@ -197,10 +196,9 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/ubuntu-prerequisites - - uses: ./.github/actions/build - - uses: ./.github/actions/test + - uses: ./.github/actions/build-and-test with: - wrapper: pg_virtualenv + test-wrapper: pg_virtualenv windows: strategy: From 0fa7d02a8f78981bd0d11fb3256e5e751b910857 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:07:31 -0700 Subject: [PATCH 06/11] Revert `.gitignore` --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index eee5413b6..b8e1ea0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ build -!.github/actions/build docs/html docs/latex From 6a1a68709fc5ae88ed942e436f8ba6bf3ebbd218 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:21:52 -0700 Subject: [PATCH 07/11] Revert `actions/checkout` upgrade --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39dd43cfe..41df418eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - 5.4 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - uses: ./.github/actions/macos-prerequisites - uses: ./.github/actions/build-and-test @@ -194,7 +194,7 @@ jobs: postgresql_version: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test with: From 153daca150cc22468e118a790d7c05dce236ecf0 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 10:31:24 -0700 Subject: [PATCH 08/11] Fix old `windows-package` job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41df418eb..cb943541c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: shell: pwsh - name: Get test data run: (new-object net.webclient).DownloadFile($env:OSMURL, "testfile.osm.bz2") + shell: pwsh - name: Exceute osm2pgsql run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.bz2 shell: bash From a39b1e53394c9f462de45ab3e63657b2eaceb73c Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 11:17:25 -0700 Subject: [PATCH 09/11] Revert `formatting` changes --- .github/actions/build-and-test/action.yml | 135 +++++++++--------- .../actions/ubuntu-prerequisites/action.yml | 25 ++-- .github/workflows/ci.yml | 4 +- 3 files changed, 81 insertions(+), 83 deletions(-) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index 1205c5b46..3aa81c17e 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -1,82 +1,81 @@ ---- -name: Build and test osm2pgsql +name: 'Build and test osm2pgsq' inputs: test-wrapper: - description: Wrapper to use around tests + description: 'Wrapper to use around tests' required: false runs: - using: composite + using: "composite" - steps: - - name: create build directory - run: mkdir build - shell: bash + steps: + - name: create build directory + run: mkdir build + shell: bash - - name: configure - run: | - CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" + - name: configure + run: | + CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" - if [ -z "${LUA_VERSION:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUA=OFF" - else - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUAJIT=${LUAJIT_OPTION:-OFF}" - fi - if [ -n "${USE_PROJ_LIB:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DUSE_PROJ_LIB=${USE_PROJ_LIB}" - fi - if [ -n "${CPP_VERSION:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_CXX_STANDARD=${CPP_VERSION}" - fi - if [ -n "${BUILD_TESTS:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_TESTS=${BUILD_TESTS}" - fi - if [ -n "${BUILD_TYPE:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - fi - if [ -n "${INSTALL_PREFIX:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" - fi - cmake \ - ${CMAKE_OPTIONS} \ - .. - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: build + if [ -z "${LUA_VERSION:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUA=OFF" + else + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUAJIT=${LUAJIT_OPTION:-OFF}" + fi + if [ -n "${USE_PROJ_LIB:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DUSE_PROJ_LIB=${USE_PROJ_LIB}" + fi + if [ -n "${CPP_VERSION:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_CXX_STANDARD=${CPP_VERSION}" + fi + if [ -n "${BUILD_TESTS:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_TESTS=${BUILD_TESTS}" + fi + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" + fi + if [ -n "${INSTALL_PREFIX:-}" ]; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" + fi + cmake \ + ${CMAKE_OPTIONS} \ + .. + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build - - name: build - run: | - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS:-}" + - name: build + run: | + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS:-}" - if [ -n "${BUILD_PARALLEL_LEVEL:-}" ]; then - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --parallel ${BUILD_PARALLEL_LEVEL}" - fi - if [ -n "${BUILD_TARGET:-}" ]; then - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --target ${BUILD_TARGET}" - fi - if [ -n "${BUILD_TYPE:-}" ]; then - CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --config ${BUILD_TYPE}" - fi - cmake --build . ${CMAKE_BUILD_OPTIONS} - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: build + if [ -n "${BUILD_PARALLEL_LEVEL:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --parallel ${BUILD_PARALLEL_LEVEL}" + fi + if [ -n "${BUILD_TARGET:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --target ${BUILD_TARGET}" + fi + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_BUILD_OPTIONS="${CMAKE_BUILD_OPTIONS} --config ${BUILD_TYPE}" + fi + cmake --build . ${CMAKE_BUILD_OPTIONS} + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build - - name: test - env: - WRAPPER: ${{ inputs.test-wrapper }} - run: | - CTEST_CMD="ctest --output-on-failure" + - name: test + env: + WRAPPER: ${{ inputs.test-wrapper }} + run: | + CTEST_CMD="ctest --output-on-failure" - if [ -n "${BUILD_TYPE:-}" ]; then - CTEST_CMD="${CTEST_CMD} --build-config ${BUILD_TYPE}" - fi + if [ -n "${BUILD_TYPE:-}" ]; then + CTEST_CMD="${CTEST_CMD} --build-config ${BUILD_TYPE}" + fi - if [ "${WRAPPER}" == "pg_virtualenv" ]; then - ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} - elif [ -n "${WRAPPER}" ]; then - ${WRAPPER} ${CTEST_CMD} - else - ${CTEST_CMD} - fi - shell: bash --noprofile --norc -euxo pipefail {0} - working-directory: build + if [ "${WRAPPER}" == "pg_virtualenv" ]; then + ${WRAPPER} -v ${POSTGRESQL_VERSION} ${CTEST_CMD} + elif [ -n "${WRAPPER}" ]; then + ${WRAPPER} ${CTEST_CMD} + else + ${CTEST_CMD} + fi + shell: bash --noprofile --norc -euxo pipefail {0} + working-directory: build diff --git a/.github/actions/ubuntu-prerequisites/action.yml b/.github/actions/ubuntu-prerequisites/action.yml index 2e9c97975..db8673d3e 100644 --- a/.github/actions/ubuntu-prerequisites/action.yml +++ b/.github/actions/ubuntu-prerequisites/action.yml @@ -1,20 +1,19 @@ ---- -name: Install Prerequisites on Ubuntu +name: 'Install Prerequisites on Ubuntu' runs: - using: composite + using: "composite" steps: - name: Remove preinstalled postgresql run: | - sudo apt-get purge -qq postgresql* + sudo apt-get remove -yq postgresql* sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo apt-get update -qq - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash - name: Install software run: | - sudo apt-get install -qq --no-install-suggests --no-install-recommends \ + sudo apt-get install -yq --no-install-suggests --no-install-recommends \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ @@ -32,7 +31,7 @@ runs: python3-setuptools \ zlib1g-dev pip3 install behave - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash - name: Install compiler and set CC/CXX run: | @@ -54,21 +53,21 @@ runs: - name: Install Lua run: | if [ -n "${LUA_VERSION:-}" ]; then - sudo apt-get install -qq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION} + sudo apt-get install -yq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION} fi - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash - name: Install LuaJIT run: | if [ "${LUAJIT_OPTION:-}" = "ON" ]; then - sudo apt-get install -qq --no-install-suggests --no-install-recommends libluajit-5.1-dev + sudo apt-get install -yq --no-install-suggests --no-install-recommends libluajit-5.1-dev fi - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash - name: Adapt postgresql configuration run: | echo 'fsync = off' | sudo tee /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf + echo 'synchronous_commit = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf echo 'full_page_writes = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf echo 'shared_buffers = 1GB' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf - echo 'synchronous_commit = off' | sudo tee -a /etc/postgresql/${POSTGRESQL_VERSION}/main/conf.d/local.conf - shell: bash --noprofile --norc -euxo pipefail {0} + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb943541c..512bd8942 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ ---- name: CI -on: [push, pull_request] +on: [ push, pull_request ] defaults: run: @@ -273,3 +272,4 @@ jobs: - name: Exceute osm2pgsql run: ./osm2pgsql-bin/osm2pgsql --slim -d osm testfile.osm.bz2 shell: bash + From 1def47ce1fc555d00e30b96aa49097ad4d4b39ac Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 11:21:00 -0700 Subject: [PATCH 10/11] Missing space --- .github/actions/build-and-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index 3aa81c17e..b6761c49c 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -75,7 +75,7 @@ runs: elif [ -n "${WRAPPER}" ]; then ${WRAPPER} ${CTEST_CMD} else - ${CTEST_CMD} + ${CTEST_CMD} fi shell: bash --noprofile --norc -euxo pipefail {0} working-directory: build From 353583d80481dfca977625592a9be176a526d2d6 Mon Sep 17 00:00:00 2001 From: David Hummel <6109326+hummeltech@users.noreply.github.com> Date: Wed, 15 Jun 2022 11:28:07 -0700 Subject: [PATCH 11/11] Revert more `formatting` changes --- .github/actions/build-and-test/action.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/actions/build-and-test/action.yml b/.github/actions/build-and-test/action.yml index b6761c49c..bee2baec3 100644 --- a/.github/actions/build-and-test/action.yml +++ b/.github/actions/build-and-test/action.yml @@ -16,31 +16,28 @@ runs: - name: configure run: | CMAKE_OPTIONS="${CMAKE_OPTIONS:--LA}" - if [ -z "${LUA_VERSION:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUA=OFF" + CMAKE_OPTIONS="$CMAKE_OPTIONS -DWITH_LUA=OFF" else - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DWITH_LUAJIT=${LUAJIT_OPTION:-OFF}" + CMAKE_OPTIONS="$CMAKE_OPTIONS -DWITH_LUAJIT=${LUAJIT_OPTION:-OFF}" fi if [ -n "${USE_PROJ_LIB:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DUSE_PROJ_LIB=${USE_PROJ_LIB}" + CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_PROJ_LIB=$USE_PROJ_LIB" fi if [ -n "${CPP_VERSION:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_CXX_STANDARD=${CPP_VERSION}" + CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_CXX_STANDARD=$CPP_VERSION" + fi + if [ -n "${BUILD_TYPE:-}" ]; then + CMAKE_OPTIONS="$CMAKE_OPTIONS -DCMAKE_BUILD_TYPE=$BUILD_TYPE" fi if [ -n "${BUILD_TESTS:-}" ]; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DBUILD_TESTS=${BUILD_TESTS}" fi - if [ -n "${BUILD_TYPE:-}" ]; then - CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" - fi if [ -n "${INSTALL_PREFIX:-}" ]; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" fi - cmake \ - ${CMAKE_OPTIONS} \ - .. - shell: bash --noprofile --norc -euxo pipefail {0} + cmake $CMAKE_OPTIONS .. + shell: bash working-directory: build - name: build