diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000000..5c657f9cb2 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,29 @@ +name: Build docker image +on: + workflow_dispatch: + pull_request: + paths: + - Dockerfile + - Makefile + - .github/workflows/docker.yaml +jobs: + build: + name: Build docker image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Build image + run: | + docker build -t libsbp-build --build-arg UID=$(id -u) - /dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ - && add-apt-repository "deb https://apt.kitware.com/ubuntu/ ${UBUNTU_RELEASE} main" \ - ) || true) \ + && add-apt-repository ppa:deadsnakes/ppa \ + && wget -O - ${KITWARE_KEY_URL} 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \ + && add-apt-repository "deb https://apt.kitware.com/ubuntu/ ${UBUNTU_RELEASE} main" \ && cat /etc/apt/sources.list \ && apt-get update \ && apt-get install -y --no-install-recommends \ @@ -65,7 +63,7 @@ RUN \ $CC $CXX \ libpython2.7-stdlib \ libpython3.8-stdlib \ - $(test $UBUNTU_RELEASE = focal && echo python-is-python3) \ + python-is-python3 \ python3-pip \ python3-setuptools \ llvm \ @@ -80,39 +78,16 @@ RUN \ graphviz \ imagemagick \ enchant \ - clang-format$(test $UBUNTU_RELEASE = bionic -o $UBUNTU_RELEASE = focal && echo -6.0) \ - # from deadsnakes - $(test $UBUNTU_RELEASE = bionic -o $UBUNTU_RELEASE = focal && echo \ - python3.5 \ - python3.6 \ - python3.7\ - python3.9 \ - ) \ - $(test $UBUNTU_RELEASE = bionic && echo python3.9-dist) \ - # from kitware or ubuntu groovy+ + clang-format-6.0 \ + python3.6 python3.6-dev python3.6-distutils \ + python3.7 python3.7-dev python3.7-distutils \ + python3 python3-dev python3-distutils \ + python3.9 python3.9-dev python3.9-distutils \ + python3.10 python3.10-dev python3.10-distutils \ + dpkg-dev \ cmake \ - && ((test $UBUNTU_RELEASE = bionic && sudo ln -s /usr/bin/pip3 /usr/bin/pip) || true) \ && curl -sSL https://get.haskellstack.org/ | sh \ - && apt remove -y \ - gnupg \ - gpg \ - netbase \ - packagekit \ - software-properties-common \ - wget \ && apt autoremove -y \ - && apt remove -y --allow-remove-essential \ - apt \ - && sudo dpkg -r --force-depends \ - dpkg-dev \ - git-man \ - fontconfig-config \ - libcommons-logging-java \ - libcommons-parent-java \ - libdpkg-perl \ - libfontbox-java \ - libpdfbox-java \ - libthai-data \ && rm -rf /var/lib/apt/lists/* /tmp/* \ && curl -s "https://get.sdkman.io" | bash \ && bash -c "source $SDKMAN_DIR/bin/sdkman-init.sh; \ diff --git a/HOWTO.md b/HOWTO.md index 64fd378cd4..7819220f6c 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -327,15 +327,8 @@ git reset --hard v # Distributing Python -The build of the libsbp wheel should be done through docker via the "manylinux" -project by running the following set of commands: - -``` -docker build -f python/Dockerfile.x86_64 -t libsbp-amd64 . -docker run -v libsbp-amd64-root:/root -v $PWD:/work --rm -it libsbp-amd64 /bin/bash -cd /work -make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=... -``` +The build of the libsbp wheel can be done via the `libsbp-build` container +described above. ## Troubleshooting diff --git a/python/.coveragerc b/python/.coveragerc deleted file mode 100644 index 44d6547605..0000000000 --- a/python/.coveragerc +++ /dev/null @@ -1,24 +0,0 @@ -[run] -branch = True -include = sbp/* - -[report] -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain about missing debug-only code: - def __repr__ - if self\.debug - - # Don't complain if non-runnable code isn't run: - if 0: - if __name__ == .__main__.: - -ignore_errors = True - -[omit] -omit = - tests/* - limbo/* - data/* \ No newline at end of file diff --git a/python/Dockerfile.arm b/python/Dockerfile.arm deleted file mode 100644 index c7780916bd..0000000000 --- a/python/Dockerfile.arm +++ /dev/null @@ -1,150 +0,0 @@ -FROM swiftnav/llvm-arm-stretch:2019.09.19 - -ARG JOBS=8 - -RUN [ "cross-build-start" ] - -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /work - -RUN \ - echo '>>> Setting up ARM build environment...' \ - && apt-get update \ - && apt-get install -y wget bzip2 build-essential python3 \ - && apt-get install -y build-essential tk-dev libncurses5-dev libncursesw5-dev \ - libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev \ - libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev \ - libffi-dev cmake ccache binutils-dev ninja-build \ - gperf gawk flex bison ncurses-dev m4 patchelf \ - texinfo help2man libpthread-stubs0-dev libtinfo-dev \ - libatlas-base-dev libsqlite3-dev tk-dev libgdbm-dev \ - libc6-dev - -RUN \ - echo '>>> Downlading get-pip.py...' \ - && wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py - -ENV PY27=2.7.18 -ENV PY27_URL=https://www.python.org/ftp/python/${PY27}/Python-${PY27}.tar.xz - -RUN \ - echo '>>> Downlading Python 2.7...' \ - && wget -O /tmp/python${PY27}.txz ${PY27_URL} - -RUN \ - echo '>>> Building Python 2.7...' \ - \ - && mkdir /tmp/python${PY27}-build && cd /tmp/python${PY27}-build \ - && tar -xJf /tmp/python${PY27}.txz \ - && cd Python-${PY27} \ - && ./configure --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib" \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - \ - && /usr/local/bin/python2.7 /tmp/get-pip.py \ - && /usr/local/bin/python2.7 -m pip install --upgrade pip \ - && /usr/local/bin/python2.7 -m pip install wheel setuptools - -ENV PY27MU=2.7.18-cp27mu -ENV CP27MU_PREFIX=/opt/cp27mu - -RUN \ - echo '>>> Building Python 2.7 (cp27mu)...' \ - \ - && mkdir /tmp/python${PY27MU}-build && cd /tmp/python${PY27MU}-build \ - && tar -xJf /tmp/python${PY27}.txz \ - && cd Python-${PY27} \ - && ./configure \ - --enable-unicode=ucs4 \ - --prefix=${CP27MU_PREFIX} \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - \ - && ${CP27MU_PREFIX}/bin/python2.7 /tmp/get-pip.py \ - && ${CP27MU_PREFIX}/bin/python2.7 -m pip install --upgrade pip \ - && ${CP27MU_PREFIX}/bin/python2.7 -m pip install wheel setuptools - -ENV PY35=3.5.10 -ENV PY35_URL=https://www.python.org/ftp/python/${PY35}/Python-${PY35}.tar.xz - -RUN \ - echo '>>> Building Python 3.5...' \ - && wget -O /tmp/python${PY35}.txz ${PY35_URL} \ - && mkdir /tmp/python${PY35}-build \ - && cd /tmp/python${PY35}-build \ - && tar -xJf /tmp/python${PY35}.txz \ - && cd Python-${PY35} \ - && ./configure --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib" \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - \ - && /usr/local/bin/python3.5 -m pip install --upgrade pip \ - && /usr/local/bin/python3.5 -m pip install wheel setuptools - -ENV PY36=3.6.12 -ENV PY36_URL=https://www.python.org/ftp/python/${PY36}/Python-${PY36}.tar.xz - -RUN \ - echo '>>> Building Python 3.6...' \ - && wget -O /tmp/python${PY36}.txz ${PY36_URL} \ - && mkdir /tmp/python${PY36}-build && cd /tmp/python${PY36}-build \ - && tar -xJf /tmp/python${PY36}.txz \ - && cd Python-${PY36} \ - && ./configure --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib" \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - \ - && /usr/local/bin/python3.6 -m pip install --upgrade pip \ - && /usr/local/bin/python3.6 -m pip install wheel setuptools - -ENV PY37=3.7.8 -ENV PY37_URL=https://www.python.org/ftp/python/${PY37}/Python-${PY37}.tar.xz - -RUN \ - echo '>>> Building Python 3.7...' \ - && wget -O /tmp/python${PY37}.txz ${PY37_URL} \ - && mkdir /tmp/python${PY37}-build && cd /tmp/python${PY37}-build \ - && tar -xJf /tmp/python${PY37}.txz \ - && cd Python-${PY37} \ - && ./configure --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib" \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - && /usr/local/bin/python3.7 -m pip install --upgrade pip \ - && /usr/local/bin/python3.7 -m pip install wheel setuptools - -ENV PY38=3.8.5 -ENV PY38_URL=https://www.python.org/ftp/python/${PY38}/Python-${PY38}.tar.xz - -RUN \ - echo '>>> Building Python 3.8...' \ - && wget -O /tmp/python${PY38}.txz ${PY38_URL} \ - && mkdir /tmp/python${PY38}-build && cd /tmp/python${PY38}-build \ - && tar -xJf /tmp/python${PY38}.txz \ - && cd Python-${PY38} \ - && ./configure --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib" \ - && make -j${JOBS} \ - && make altinstall \ - && cd /work \ - && /usr/local/bin/python3.8 -m pip install --upgrade pip \ - && /usr/local/bin/python3.8 -m pip install wheel setuptools - -RUN \ - rm /tmp/python${PY27}.txz \ - && rm -rf /tmp/python${PY27}-build \ - && rm /tmp/python${PY35}.txz \ - && rm -rf /tmp/python${PY35}-build \ - && rm /tmp/python${PY36}.txz \ - && rm -rf /tmp/python${PY36}-build \ - && rm /tmp/python${PY37}.txz \ - && rm -rf /tmp/python${PY37}-build \ - && rm /tmp/python${PY38}.txz \ - && rm -rf /tmp/python${PY38}-build \ - && rm /tmp/get-pip.py - -RUN [ "cross-build-end" ] diff --git a/python/Dockerfile.llvm_arm b/python/Dockerfile.llvm_arm deleted file mode 100644 index f80b162718..0000000000 --- a/python/Dockerfile.llvm_arm +++ /dev/null @@ -1,50 +0,0 @@ -FROM balenalib/armv7hf-debian:stretch-build - -RUN [ "cross-build-start" ] - -ENV DEBIAN_FRONTEND=noninteractive - -WORKDIR /work - -RUN \ - echo '>>> Setting up ARM build environment...' \ - && apt-get update \ - && apt-get install -y wget bzip2 build-essential python3 \ - && apt-get install -y build-essential tk-dev libncurses5-dev libncursesw5-dev \ - libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev \ - libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev \ - libffi-dev cmake ccache binutils-dev ninja-build \ - gperf gawk flex bison ncurses-dev m4 patchelf \ - texinfo help2man libpthread-stubs0-dev libtinfo-dev \ - libatlas-base-dev libsqlite3-dev tk-dev libgdbm-dev \ - libc6-dev - -ENV LLVM_BRANCH=release_70 -ENV LLVM_URL=https://github.com/llvm-mirror/llvm.git -ENV CLANG_URL=https://github.com/llvm-mirror/clang.git - -RUN \ - echo '>>> Building LLVM 7.0...' \ - && git clone --depth=1 --single-branch -b ${LLVM_BRANCH} ${LLVM_URL} /tmp/llvm \ - && git clone --depth=1 --single-branch -b ${LLVM_BRANCH} ${CLANG_URL} /tmp/llvm/tools/clang \ - && mkdir -p /tmp/llvm/build \ - && cd /tmp/llvm/build \ - && cmake -G Ninja /tmp/llvm \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DLLVM_TARGETS_TO_BUILD=ARM \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BINUTILS_INCDIR=/usr/include \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_TERMINFO=0 \ - -DCMAKE_C_COMPILER=/usr/bin/cc \ - -DCMAKE_CXX_COMPILER=/usr/bin/c++ \ - -DCMAKE_ASM_COMPILER=/usr/bin/as \ - -DCMAKE_CXX_FLAGS="-I/root/llvm/build/tools/clang/include -I/root/llvm/tools/clang/include -DENDIAN_LITTLE=1" \ - && ninja -v \ - && ninja install - -RUN \ - echo '>>> Cleaning up...' \ - && rm -rf /tmp/llvm - -RUN [ "cross-build-end" ] diff --git a/python/Dockerfile.win b/python/Dockerfile.win deleted file mode 100644 index 2112ee1d3d..0000000000 --- a/python/Dockerfile.win +++ /dev/null @@ -1,41 +0,0 @@ -# escape=` - -FROM mcr.microsoft.com/windows:1903 - -SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] - -RUN ` - Set-ExecutionPolicy Bypass -Scope Process -Force; ` - [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; ` - iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -RUN ` - choco install -y miniconda3; ` - choco install -y make; ` - choco install -y vcredist15; ` - choco install -y vcredist2019; ` - choco install -y git - -RUN ["choco", "install", "-y", "visualstudio2019-workload-vctools"] - -USER ContainerAdministrator - -RUN Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force - -SHELL ["CMD.EXE", "/C"] -RUN setx /M PATH "%PATH%;c:\tools\miniconda3;c:\tools\miniconda3\condabin" - -SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -RUN Set-ExecutionPolicy RemoteSigned - -USER ContainerUser - -RUN ` - Install-Module posh-vs -Force -Verbose -Scope CurrentUser; ` - conda init - -WORKDIR C:/work - -ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] - -# vim: ft=dockerfile diff --git a/python/Dockerfile.x86_64 b/python/Dockerfile.x86_64 deleted file mode 100644 index 43dd8908c6..0000000000 --- a/python/Dockerfile.x86_64 +++ /dev/null @@ -1,10 +0,0 @@ -FROM quay.io/pypa/manylinux2010_x86_64 - -RUN \ - echo Setting up Linux x86 build environment... \ - && yum install -y wget \ - && wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && bash /tmp/miniconda.sh -b \ - && rm /tmp/miniconda.sh - -ENV PATH=/root/miniconda3/bin:$PATH diff --git a/python/test_requirements.txt b/python/test_requirements.txt index 513084adfe..940b52000f 100644 --- a/python/test_requirements.txt +++ b/python/test_requirements.txt @@ -1,6 +1,7 @@ pytest pytest-cov pytest-flakes +pytest-rerunfailures coverage tox pyyaml diff --git a/python/tox.ini b/python/tox.ini index 1f1382cb19..6b2cf02fe5 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,10 +1,16 @@ [tox] -envlist = py{36,37,38,39,310}-{sbp2json,noextra}-construct{2_9_33,2_9_52,2_10_53,,2_10_67} +envlist = clean,py{36,37,38,39,310}-{sbp2json,noextra}-construct{2_9_33,2_9_52,2_10_53,,2_10_67} minversion = 1.7.2 -install_command=python -m pip install --no-use-pep517 {opts} {packages} + +[testenv:clean] +deps = coverage +skip_install = true +commands = coverage erase [testenv] extras = sbp2json: sbp2json +depends = + py{36,37,38,39,310}-{sbp2json,noextra}-construct{2_9_33,2_9_52,2_10_53,,2_10_67}: clean deps = construct_2_9_33: construct==2.9.33 construct_2_9_52: construct==2.9.52 @@ -16,6 +22,32 @@ deps = passenv = PYTEST_ADDOPTS commands = - py.test -v --cov --cov-report term-missing sbp tests + py.test -v --reruns 5 --cov --cov-append --cov-report=term-missing sbp tests {toxinidir}/../test_data/sanity.sh sitepackages = False + +[coverage:run] +branch = True +include = sbp/* +parallel = true +data_file = .coverage.{toxenv} + +[coverage:report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: +ignore_errors = True + +[coverage:omit] +omit = + tests/* + limbo/* + data/*