WIP: New implementation of kealib python bindings using libawkward (#12)

* initial work on libawkward support

* work implementation

* extra checks and remove old source files

* progress on building neighbours from an array

* tidy up and docstrings

* tidy dockerfiles

* added test for the accumulator and fixed a few bugs

* eightConnected -> fourConnected

* tidy up and tests

* add main script and get install working

* more comments plus add function to add a bool column

* change to installing under 'kea'. Add a conda build

* fix Windows build and move to kealib namespace

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "python/pybind11"]
+	path = python/pybind11
+	url = https://github.com/pybind/pybind11.git

python/CMakeLists.txt

@@ -0,0 +1,59 @@
+#  Permission is hereby granted, free of charge, to any person 
+#  obtaining a copy of this software and associated documentation 
+#  files (the "Software"), to deal in the Software without restriction, 
+#  including without limitation the rights to use, copy, modify, 
+#  merge, publish, distribute, sublicense, and/or sell copies of the 
+#  Software, and to permit persons to whom the Software is furnished 
+#  to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be 
+#  included in all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 
+#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
+#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 
+#  ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 
+#  CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
+#  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+cmake_minimum_required(VERSION 3.1...3.16)
+project(extrat LANGUAGES CXX)
+
+if(MSVC)
+  set(LIBKEA_NAME "libkea")
+  set(LIBKEA_CONFIG "kea-config.bat")
+else()
+  set(LIBKEA_NAME "kea")
+  set(LIBKEA_CONFIG "kea-config")
+endif()
+
+# get libawkward info
+execute_process(COMMAND python -m awkward.config --incdir OUTPUT_VARIABLE AWKWARD_INCLUDE OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND python -m awkward.config --libdir OUTPUT_VARIABLE AWKWARD_LIBRARIES OUTPUT_STRIP_TRAILING_WHITESPACE)
+# get kealib info
+execute_process(COMMAND ${LIBKEA_CONFIG} --includes OUTPUT_VARIABLE KEALIB_INCLUDE OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${LIBKEA_CONFIG} --ldflags OUTPUT_VARIABLE KEALIB_LIBRARIES OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+find_package(GDAL REQUIRED)
+find_package(Python3 COMPONENTS Interpreter Development NumPy)
+
+message(STATUS "Includes: ${AWKWARD_INCLUDE} ${KEALIB_INCLUDE} ${GDAL_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}")
+include_directories(BEFORE "${AWKWARD_INCLUDE}" "${KEALIB_INCLUDE}" "${GDAL_INCLUDE_DIRS}" "${Python3_NumPy_INCLUDE_DIRS}")
+
+add_subdirectory(pybind11)
+
+find_library(CPU-KERNELS awkward-cpu-kernels REQUIRED HINTS ${AWKWARD_LIBRARIES})
+find_library(LIBAWKWARD awkward REQUIRED HINTS ${AWKWARD_LIBRARIES})
+find_library(LIBKEA ${LIBKEA_NAME} REQUIRED HINTS ${KEALIB_LIBRARIES})
+message(STATUS "Libraries: ${CPU-KERNELS} ${LIBAWKWARD} ${LIBKEA} ${GDAL_LIBRARIES}")
+
+pybind11_add_module(extrat extrat.cpp)
+target_compile_definitions(extrat PRIVATE NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION)
+set_target_properties(extrat PROPERTIES CXX_VISIBILITY_PRESET hidden)
+target_link_libraries(extrat PRIVATE ${CPU-KERNELS} ${LIBAWKWARD} ${LIBKEA} ${GDAL_LIBRARIES})
+
+install(FILES kea_build_neighbours.py DESTINATION bin)
+set(PYKEA_INSTALL_DIR "${Python3_SITELIB}/kealib" CACHE PATH "Install Dir for Python Bindings")
+install(TARGETS extrat DESTINATION ${PYKEA_INSTALL_DIR})
+install(FILES __init__.py DESTINATION ${PYKEA_INSTALL_DIR})

python/Dockerfiles/dockerimg

@@ -0,0 +1,7 @@
+# Simple docker image that is the conda build image
+# Plus the conda packages required to build pykealib
+
+FROM condaforge/linux-anvil-comp7
+
+RUN /opt/conda/bin/tini -- /opt/docker/bin/entrypoint bash -c "conda create -n mybuild -c conda-forge python=3.8 gxx_linux-64 cmake awkward gdal=3.2"
+

python/Dockerscripts/builddockerimage.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# This script builds the docker image that is used 
+# for compilation.
+
+# Run this first
+
+# Needs to be run as root
+
+docker build -t awk:latest -f Dockerfiles/dockerimg .
+

python/Dockerscripts/buildpackage.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# This script builds the package using Docker
+
+# Run this second
+
+# Needs to be run as root
+
+BUILD_ROOT=$(cd "$(dirname "$0")/../.."; pwd;)
+export HOST_USER_ID=$(id -u)
+# needed so we can write the output...
+chmod a+w .
+
+docker run -it \
+    -v "${BUILD_ROOT}":/home/conda/recipe_root:rw \
+    awk:latest \
+    bash \
+    /home/conda/recipe_root/python/Dockerscripts/dockerbuildconda.sh

python/Dockerscripts/dockerbuild.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Simple script that does the building in a conda-like environment
+# I've copied all the env vars accross that conda uses
+
+# Called buy buildpackage.sh
+
+set -e
+
+echo $USER
+. /opt/conda/etc/profile.d/conda.sh
+conda activate mybuild
+
+export BUILD_PREFIX=$CONDA_PREFIX
+export PREFIX=$CONDA_PREFIX
+export AR=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-ar
+export AS=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-as
+export CXXFILT=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-c++filt
+export ELFEDIT=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-elfedit
+export GPROF=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-gprof
+export LD_GOLD=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-ld.gold
+export LD=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-ld
+export NM=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-nm
+export OBJCOPY=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-objcopy
+export OBJDUMP=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-objdump
+export RANLIB=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-ranlib
+export READELF=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-readelf
+export SIZE=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-size
+export STRINGS=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-strings
+export STRIP=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-strip
+export CC=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-cc
+export CFLAGS="-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem $PREFIX/include -fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/qgis-3.10.13 -fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix"
+export CMAKE_ARGS="-DCMAKE_LINKER=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-ld -DCMAKE_STRIP=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-strip -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_FIND_ROOT_PATH=$PREFIX;$BUILD_PREFIX/x86_64-conda-linux-gnu/sysroot -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_INSTALL_LIBDIR=lib"
+export CMAKE_PREFIX_PATH=$PREFIX:$BUILD_PREFIX/x86_64-conda-linux-gnu/sysroot/usr
+export CONDA_BUILD_SYSROOT=$BUILD_PREFIX/x86_64-conda-linux-gnu/sysroot
+export _CONDA_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_x86_64_conda_cos6_linux_gnu
+export CPPFLAGS="-DNDEBUG -D_FORTIFY_SOURCE=2 -O2 -isystem $PREFIX/include"
+export CPP=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-cpp
+export DEBUG_CFLAGS="-march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem $PREFIX/include -fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/qgis-3.10.13 -fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix"
+export DEBUG_CPPFLAGS="-D_DEBUG -D_FORTIFY_SOURCE=2 -Og -isystem $BUILD_PREFIX/include"
+export GCC_AR=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-gcc-ar
+export GCC=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-gcc
+export GCC_NM=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-gcc-nm
+export GCC_RANLIB=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-gcc-ranlib
+export host_alias=x86_64-conda-linux-gnu
+export HOST=x86_64-conda-linux-gnu
+export LDFLAGS="-Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,--gc-sections -Wl,-rpath,$PREFIX/lib -Wl,-rpath-link,$PREFIX/lib -L$PREFIX/lib"
+export CXXFLAGS="-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem $PREFIX/include -fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/qgis-3.10.13 -fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix"
+export CXX_FOR_BUILD=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-c++
+export CXX=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-c++
+export DEBUG_CXXFLAGS="-fvisibility-inlines-hidden -std=c++17 -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-all -fno-plt -Og -g -Wall -Wextra -fvar-tracking-assignments -ffunction-sections -pipe -isystem $PREFIX/include -fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/qgis-3.10.13 -fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix"
+export GXX=$BUILD_PREFIX/bin/x86_64-conda-linux-gnu-g++
+
+cd /home/conda/recipe_root/python
+rm -rf build
+mkdir build
+cd build
+
+cmake -D CMAKE_INSTALL_PREFIX=$CONDA_PREFIX -D CMAKE_BUILD_TYPE=Release ..
+make
+make install

python/Dockerscripts/dockerbuildconda.sh

@@ -0,0 +1,8 @@
+
+. /opt/conda/etc/profile.d/conda.sh
+conda activate mybuild
+
+cd /home/conda/recipe_root/python
+
+conda build --output-folder=. --python=3.8 condarecipe
+

python/__init__.py

@@ -0,0 +1,6 @@
+"""
+Module for Kealib 'extras' - functionality that GDAL 
+doesn't currently support
+"""
+# load this into mem as symbols are required by the shared libs in here
+import awkward

python/condarecipe/build.sh

@@ -0,0 +1,11 @@
+
+mkdir -p build
+cd build
+
+cmake -D CMAKE_INSTALL_PREFIX=$PREFIX \
+    -D CMAKE_BUILD_TYPE=Release \
+	-D CMAKE_PREFIX_PATH=$PREFIX \
+	-D Python3_EXECUTABLE=$PYTHON \
+	..
+make
+make install

python/condarecipe/meta.yaml

@@ -0,0 +1,31 @@
+package:
+  name: pykealib
+  version: "0.1"
+
+source:
+  path: ..
+
+build:
+  number: 1
+
+requirements:
+  build:
+    - cmake
+    - make  # [unix]
+    - {{ compiler('cxx') }}
+  host:
+    - python
+    - gdal 3.2.*
+    - numpy 1.20.*
+    - kealib
+    - awkward
+  run:
+    - python
+    - gdal 3.2.*
+    - numpy 1.20.*
+    - kealib
+    - awkward
+
+test:
+  imports:
+    - kealib.extrat