Drop "bindgen" from libthemis-sys dependencies (#626)

* Drop "bindgen" from libthemis-sys dependencies

Initially it was used to dynamically generate Rust bindings to C code.
I have thought that this is the most idiomatic way to it. It is also
more robust as Bindgen will be working with the actual Themis version
installed on the user system.

However, Bindgen has a lot of transitive dependencies, they take quite a
while to compile, and they need to be compiled by our users too. This is
a constant source of irritation for developers. Users are not really
happy with that as well.

While we try our best to maintain backwards compatibility when it comes
to symbols and dependencies, we generally do not bother with forward
compatibility. And we effectively pin wrappers to the core library.
That is, RustThemis 0.N is guaranteed to work with Themis Core 0.N,
it is also guaranteed to work with Themis Core 0.N+1, and will most
likely work with Themis Core 0.N+2 (unless there is deprecated API
removed). However, the converse is not the case. RustThemis 0.N+1 may
work with Themis 0.N if no new features were added, but otherwise it
might fail to compile.

Thus, RustThemis is effectivelly pinned to Themis Core version, meaning
that their versions must match. If that's the case, we do not need to
generate FFI bindings on the user machine because we know which version
we are shipping with and what symbols are available there. Thankfully,
we also write portable C code so Themis API does not depend on the
target architecture -- which allows us to ship the bindings verbatim.
Otherwise we'd have no choice other that generate them on the fly.

With that in mind, start generating "src/lib.rs" manually with the help
of "bindgen.sh" script. The developers are expected to run it on their
machine to regenerate bindings somewhere before release. The users now
do not need to generate these bindings on their machine, meaning that
we can drop "bindgen" crate dependency and all supporting code.

In the end, this gives *significant* improvements in transitive
dependency count (down to 3, from 43) as well as *ENORMOUS*
improvements in build times:

           Before  After
    Linux   28.43   2.68  Debug
    macOS   52.80   8.47

    Linux   52.08   5.35  Release
    macOS  127.21   8.71

CI builds were a source of pain since they build RustThemis in both
debug and release versions, possibly several times, and in the worst
case they are running with only one virtual CPU so the absolute time
to run the test suite might be several minutes.

* Check "./bindgen.sh" output on CI

Set up a GitHub Actions step to verify that the output of "./bindgen.sh"
script does not change and "src/lib.rs" does not need an update. This
will break the build once changes are necessary. Typically this means
that some new API is exported by Themis or there are some changes in
Bindgen's behavior. Either way we would like to be notified.

.github/workflows/test-rust.yaml

@@ -168,3 +168,63 @@ jobs:
           sleep 1
           test "$(cat server.txt)" = "[-] no match"
           test "$(cat client.txt)" = "[-] no match"
+
+  bindgen:
+    name: libthemis-sys bindings
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install system dependencies
+        run: |
+          sudo sh -c 'echo "DEBIAN_FRONTEND=noninteractive" >> /etc/environment'
+          sudo apt update
+          sudo apt install --yes libssl-dev llvm
+      - name: Install stable Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          components: rustfmt
+      # Cargo pulls in quite a few stuff from the Internet and Rust always
+      # (slowly) recompiles dependencies, so make heavy use of caching
+      - name: Cache Cargo registry
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-registry-
+      - name: Cache Cargo index
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-index-
+      - name: Cache Cargo build
+        uses: actions/cache@v1
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-unit-tests-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-build-target-unit-tests-
+            ${{ runner.os }}-cargo-build-target-
+      - name: Install Bindgen
+        run: cargo install bindgen
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Check bindgen.sh output
+        run: |
+          cd src/wrappers/themis/rust/libthemis-sys
+          mv src/lib.rs src/lib.rs.old
+          ./bindgen.sh
+          mv src/lib.rs src/lib.rs.new
+          if ! diff -u src/lib.rs.old src/lib.rs.new
+          then
+              echo
+              echo "Something has changed in exported definitions (see above)."
+              echo "You may need to do"
+              echo
+              echo "    cd src/wrappers/themis/rust/libthemis-sys"
+              echo "    ./bindgen.sh"
+              echo
+              echo "to refresh src/lib.rs content."
+              exit 1
+          fi

CHANGELOG.md

@@ -478,6 +478,7 @@ _Code:_
 - **Rust**
 
   - New object `themis::keys::SymmetricKey` can be used to generate symmetric keys for Secure Cell ([#561](https://github.com/cossacklabs/themis/pull/561)).
+  - Significantly reduced compilation time by removing `bindgen` crate from dependencies ([#626](https://github.com/cossacklabs/themis/pull/626)).
 
 - **WebAssembly**
 

src/wrappers/themis/rust/CHANGELOG.md

@@ -9,6 +9,11 @@ The version currently in development.
 
 [#561]: https://github.com/cossacklabs/themis/pull/561
 
+## Internal improvements
+
+- Significantly reduced compilation time by removing `bindgen` crate from dependencies.
+  ([#626](https://github.com/cossacklabs/themis/pull/626))
+
 Version 0.12.0 - 2019-09-26
 ===========================
 

src/wrappers/themis/rust/libthemis-sys/Cargo.toml

@@ -20,7 +20,6 @@ maintenance = { status = "actively-developed" }
 vendored = ["libthemis-src"]
 
 [build-dependencies]
-bindgen = "0.46.0"
 cc = "1.0.28"
 libthemis-src = { path = "../libthemis-src", version = "0.12.0", optional = true }
 pkg-config = "0.3.14"

src/wrappers/themis/rust/libthemis-sys/bindgen.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Generate "src/lib.rs" file with bindings
+#
+# Run this script from "libthemis-sys" directory before release
+# to update raw FFI bindings with newly added functions and types:
+#
+#     ./bindgen.sh
+#
+# You need to have Bindgen, LLVM, rustfmt installed to run this script.
+# Bindgen can be installed with
+#
+#     cargo install bindgen
+#
+# rustfmt can be installed with
+#
+#     rustup component add rustfmt
+#
+# Suitable LLVM can usually be installed from your system's repositories.
+
+set -e -o pipefail
+
+# This is a pattern for what we export from libthemis-sys.
+# Bindgen sees Soter as well as some system libraries, we don't need that.
+WHITELIST="(THEMIS|themis|secure_(comparator|session)|STATE)_.*"
+
+# Currently, we don't pass --target since none of the symbols we're linking
+# against are architecture-specific. If this ever becomes a problem, then the
+# thing to do is to split the generated code into different files for different
+# platforms (like themis_x86_64.rs, themis_arm64.rs, etc.) and conditionally
+# compile them depending on target.
+bindgen bindgen.h \
+    --no-doc-comments \
+    --rustified-enum "themis_key_kind" \
+    --size_t-is-usize \
+    --whitelist-function "$WHITELIST" \
+    --whitelist-type "$WHITELIST" \
+    --whitelist-var "$WHITELIST" \
+    --output src/lib.rs \
+    -- \
+    -I ../../../.. # ${repository_root}/src
+
+TMP="$(mktemp)"
+
+# Prepend copyright comment, #[allow] for various warnings we don't care about.
+(cat << EOF
+// Copyright 2018 (c) rust-themis developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Raw FFI bindings to libthemis.
+
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+// For some weird reasons Clippy gets run on this crate as it's a path dependency of themis.
+// This should not happen (see https://github.com/rust-lang-nursery/rust-clippy/issues/1066)
+// but until that's fixed again, disable all lints which get triggered by the code generated
+// by bindgen.
+#![allow(clippy::all)]
+
+EOF
+
+cat src/lib.rs) \
+| rustfmt > "$TMP"
+mv "$TMP" src/lib.rs

src/wrappers/themis/rust/libthemis-sys/build.rs

@@ -12,37 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::env;
-use std::path::{Path, PathBuf};
-
 fn main() {
-    let themis = get_themis();
-
-    let whitelist = "(THEMIS|themis|secure_(comparator|session)|STATE)_.*";
-    let bindings = bindgen::Builder::default()
-        .clang_args(clang_include_paths(&themis))
-        .clang_args(clang_library_paths(&themis))
-        .header("src/wrapper.h")
-        .whitelist_function(whitelist)
-        .whitelist_type(whitelist)
-        .whitelist_var(whitelist)
-        .rustified_enum("themis_key_kind")
-        .generate()
-        .expect("generating bindings");
-
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    bindings
-        .write_to_file(out_path.join("bindings.rs"))
-        .expect("writing bindings!");
-}
-
-struct Library {
-    include_paths: Vec<PathBuf>,
-    link_paths: Vec<PathBuf>,
-}
-
-/// Embarks on an incredible adventure and returns with a suitable Themis (or dies trying).
-fn get_themis() -> Library {
     #[cfg(feature = "vendored")]
     libthemis_src::make();
 
@@ -53,10 +23,10 @@ fn get_themis() -> Library {
     pkg_config.statik(true);
 
     match pkg_config.probe("libthemis") {
-        Ok(library) => Library {
-            include_paths: library.include_paths,
-            link_paths: library.link_paths,
-        },
+        Ok(_) => {
+            // pkg_config has already printed instructions for cargo.
+            // We're done here.
+        }
         Err(error) => {
             eprintln!(
                 "
@@ -78,80 +48,35 @@ variable to the path where `libthemis.pc` file is located.
             );
             eprintln!("{}", error);
 
-            if let Some(library) = try_system_themis() {
+            if try_system_themis() {
                 eprintln!(
                     "\
 `libthemis-sys` tried using standard system paths and it seems that Themis
 is available on your system. (However, pkg-config failed to find it.)
 We will link against the system library.
 "
                 );
-                return library;
             } else {
                 eprintln!(
                     "\
 `libthemis-sys` also tried to use standard system paths, but without success.
 It seems that Themis is really not installed in your system.
 "
                 );
+                panic!("Themis Core not installed");
             }
-
-            panic!("Themis Core not installed");
         }
     }
 }
 
-fn clang_include_paths(library: &Library) -> Vec<String> {
-    library
-        .include_paths
-        .iter()
-        .map(|p| format!("-I{}", p.display()))
-        .collect()
-}
-
-fn clang_library_paths(library: &Library) -> Vec<String> {
-    library
-        .link_paths
-        .iter()
-        .map(|p| format!("-L{}", p.display()))
-        .collect()
-}
-
-trait CCBuildEx {
-    fn includes<I>(&mut self, dirs: I) -> &mut Self
-    where
-        I: IntoIterator,
-        I::Item: AsRef<Path>;
-}
-
-impl CCBuildEx for cc::Build {
-    fn includes<I>(&mut self, dirs: I) -> &mut Self
-    where
-        I: IntoIterator,
-        I::Item: AsRef<Path>,
-    {
-        for dir in dirs {
-            self.include(dir);
-        }
-        self
-    }
-}
-
-fn try_system_themis() -> Option<Library> {
+fn try_system_themis() -> bool {
     let mut build = cc::Build::new();
     build.file("src/dummy.c");
     build.flag("-lthemis");
 
-    match build.try_compile("dummy") {
-        Ok(_) => {
-            println!("cargo:rustc-link-lib=dylib=themis");
-
-            // Use only system paths for header and library lookup.
-            Some(Library {
-                include_paths: vec![],
-                link_paths: vec![],
-            })
-        }
-        Err(_) => None,
+    let result = build.try_compile("dummy");
+    if result.is_ok() {
+        println!("cargo:rustc-link-lib=dylib=themis");
     }
+    result.is_ok()
 }