feat: bring back and adapt python bindings with rust core

* import python, try to adapt for rust

* add missing wrapper functions

* - try to write up how to build python bindings
- strike some unused functions from deltachat.h

* adjustments to make tox work

* try to run circle-ci with python build

* don't do docs

* running cargo test as well

* don't run cargo test anymore, that's done in other ci jobs

* also build docs

* don't run doxygen anymore

* subst C with Rust

* a try to get better wheels

Closes #41

.circleci/config.yml

@@ -123,11 +123,53 @@ jobs:
           target: "aarch64-linux-android"
 
 
+  build_test_docs_wheel:
+    machine: True
+    steps:
+      - checkout
+      # - run: docker pull deltachat/doxygen 
+      - run: docker pull deltachat/coredeps
+      - run: 
+          name: build docs, run tests and build wheels 
+          command: ci_scripts/ci_run.sh
+          environment:
+            TESTS: 1
+            DOCS: 1
+
+      - run: 
+           name: copying docs and wheels to workspace 
+           command: |
+             mkdir -p workspace/python
+             # cp -av docs workspace/c-docs
+             cp -av python/.docker-tox/wheelhouse workspace/
+             cp -av python/doc/_build/ workspace/py-docs
+
+      - persist_to_workspace: 
+          root: workspace 
+          paths: 
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  upload_docs_wheels:
+    machine: True
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: ls -laR workspace 
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse
+
+
 workflows:
   version: 2.1
 
   test:
     jobs:
+      - build_test_docs_wheel
+      - upload_docs_wheels:
+          requires:
+              - build_test_docs_wheel
       - cargo_fetch
       - rustfmt:
           requires:

ci_scripts/README.md

@@ -0,0 +1,52 @@
+
+# Continuous Integration Scripts for Delta Chat
+
+Continuous Integration is run through CircleCI
+but is largely independent of it. 
+
+
+## Generating docker containers for performing build step work 
+
+All tests, docs and wheel building is run in docker containers:
+
+- **coredeps/Dockerfile** specifies an image that contains all 
+  of Delta Chat's core dependencies as linkable libraries. 
+  It also serves to run python tests and build wheels 
+  (binary packages for Python). 
+
+- **doxygen/Dockerfile** specifies an image that contains
+  the doxygen tool which is used to generate C-docs. 
+
+To run tests locally you can pull existing images from "docker.io",
+the hub for sharing Docker images::
+
+    docker pull deltachat/coredeps
+    docker pull deltachat/doxygen 
+
+or you can build the docker images yourself locally
+to avoid the relatively large download:: 
+
+    cd ci_scripts  # where all CI things are 
+    docker build -t deltachat/coredeps docker-coredeps
+    docker build -t deltachat/doxygen docker-doxygen 
+
+## ci_run.sh (main entrypoint called by circle-ci)
+
+Once you have the docker images available 
+you can run python testing, documentation generation 
+and building binary wheels:: 
+
+    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
+
+## ci_upload.sh (uploading artifacts on success) 
+
+- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- python wheels to `https://m.devpi.net/dc/<BRANCH>`
+  so that you install fully self-contained wheels like this:
+  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
+
+
+

ci_scripts/ci_run.sh

@@ -0,0 +1,22 @@
+# perform CI jobs on PRs and after merges to master.
+# triggered from .circleci/config.yml
+
+set -e -x
+
+export BRANCH=${CIRCLE_BRANCH:-test7}
+
+# run doxygen on c-source (needed by later doc-generation steps).
+# XXX modifies the host filesystem docs/xml and docs/html directories
+# XXX which you can then only remove with sudo as they belong to root
+
+# XXX we don't do doxygen doc generation with Rust anymore, needs to be
+# substituted with rust-docs 
+#if [ -n "$DOCS" ] ; then 
+#    docker run --rm -it -v $PWD:/mnt -w /mnt/docs deltachat/doxygen doxygen
+#fi
+
+# run everything else inside docker (TESTS, DOCS, WHEELS) 
+docker run -e BRANCH -e TESTS -e DOCS \
+           --rm -it -v $(pwd):/mnt -w /mnt \
+           deltachat/coredeps ci_scripts/run_all.sh
+

ci_scripts/ci_upload.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 1
+fi
+
+set -xe
+
+#DOXYDOCDIR=${1:?directory where doxygen docs to be found}
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+#rsync -avz \
+#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+#  "$DOXYDOCDIR/html/" \
+#  delta@py.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+devpi use dc/$BRANCH || {
+    devpi index -c $BRANCH 
+    devpi use dc/$BRANCH
+}
+devpi index $BRANCH bases=/root/pypi
+devpi upload deltachat*.whl
+
+popd

ci_scripts/docker-coredeps/Dockerfile

@@ -0,0 +1,25 @@
+FROM quay.io/pypa/manylinux1_x86_64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust nightly 
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
+
+# Install a recent Perl, needed to install OpenSSL
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+# Install OpenSSL
+ADD deps/build_openssl.sh /builder/build_openssl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
+

ci_scripts/docker-coredeps/deps/build_openssl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.28.0
+PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION} 
+
+./Configure -de
+make
+make install

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x 
+
+# Install Rust 
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-04-19 -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-04-19-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+#
+set -e -x
+
+# perform clean build of core and install 
+export TOXWORKDIR=.docker-tox
+
+# build core library
+
+cargo build --release -p deltachat_ffi
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+#
+# run python tests
+#
+
+if [ -n "$TESTS" ]; then 
+
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi

ci_scripts/docker-doxygen/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:stable
+
+# this is tagged as deltachat/doxygen 
+
+RUN apt-get update && apt-get install -y doxygen 

ci_scripts/run_all.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+# typically run in a docker container that contains all library deps
+# but should also work outside if you have the dependencies installed  
+# on your system. 
+
+set -e -x
+
+# perform clean build of core and install 
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# make sure subsequent compiler invocations find header and libraries
+export CFLAGS=-I`pwd`/deltachat-ffi
+export LD_LIBRARY_PATH=`pwd`/target/release
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then 
+
+    pushd python 
+    # prepare a clean tox run 
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox 
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37,auditwheels
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi