Merge pull request #80 from kaedroho/static-linking

Implement static linking

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "rocksdb-sys/snappy"]
+	path = rocksdb-sys/snappy
+	url = https://github.com/google/snappy.git
+[submodule "rocksdb-sys/rocksdb"]
+	path = rocksdb-sys/rocksdb
+	url = https://github.com/facebook/rocksdb.git

.travis.yml

@@ -2,37 +2,21 @@ language: rust
 dist: trusty
 sudo: true
 
-matrix:
-  include:
-    - rust: stable
-      env: ROCKSDB_VERSION=4.13
-    - rust: beta
-      env: ROCKSDB_VERSION=4.13
-    - rust: stable
-      env: ROCKSDB_VERSION=4.5
-    - rust: stable
-      env: ROCKSDB_VERSION=4.1
+rust:
+  - stable
+  - beta
 
 addons:
   apt:
     sources:
-    - ubuntu-toolchain-r-test
+      - ubuntu-toolchain-r-test
     packages:
-    - gcc-5
-    - g++-5
-    - libgflags-dev
-    - libsnappy-dev
-    - zlib1g-dev
-    - libbz2-dev
-
-install:
-  - curl -L https://github.com/facebook/rocksdb/archive/$ROCKSDB_VERSION.fb.tar.gz | tar xvz -C $HOME/rocksdb
-  - sudo INSTALL_PATH=/usr make -C $HOME/rocksdb/rocksdb-$ROCKSDB_VERSION.fb install-shared
+      - g++-5
 
 script:
+  - cargo test --manifest-path=rocksdb-sys/Cargo.toml
   - cargo test
 
 cache:
   directories:
     - $HOME/.cache/
-    - $HOME/rocksdb/

Cargo.toml

@@ -25,3 +25,4 @@ path = "test/test.rs"
 
 [dependencies]
 libc = "0.2.13"
+rocksdb-sys = { path = "rocksdb-sys", version = "0.4.0" }

README.md

@@ -18,20 +18,10 @@ This library has been tested against RocksDB 3.13.1 on linux and OSX.  The 0.4.1
   - [x] column family operations
   - [ ] prefix seek
   - [ ] slicetransform
-  - [ ] windows support
+  - [x] windows support
 
 Feedback and pull requests welcome!  If a particular feature of RocksDB is important to you, please let me know by opening an issue, and I'll prioritize it.
 
-###### Prerequisite: RocksDB
-
-First, use your system's package manager to install snappy.  This is optional, but lets rocksdb take advantage of better compression, and some code may require it.
-
-```bash
-wget https://github.com/facebook/rocksdb/archive/rocksdb-3.8.tar.gz
-tar xvf rocksdb-3.8.tar.gz && cd rocksdb-rocksdb-3.8 && make shared_lib
-sudo make install
-```
-
 ### Running
 ###### Cargo.toml
 ```rust

rocksdb-sys/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

rocksdb-sys/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "rocksdb-sys"
+version = "0.4.0"
+authors = ["Karl Hobley <karlhobley10@gmail.com>", "Arkadiy Paronyan <arkadiy@ethcore.io>"]
+license = "MIT/Apache-2.0/BSD"
+description = "Native bindings to librocksdb"
+readme = "README.md"
+repository = "https://github.com/jsgf/rocksdb-sys.git"
+keywords = [ "ffi", "rocksdb" ]
+
+build = "build.rs"
+links = "rocksdb"
+
+[features]
+default = [ "static" ]
+static = []
+
+[dependencies]
+libc = "0.2"
+
+[build-dependencies]
+gcc = { version = "0.3", features = ["parallel"] }

rocksdb-sys/Makefile

@@ -0,0 +1,6 @@
+include rocksdb/src.mk
+
+rocksdb_lib_sources.txt: rocksdb/src.mk
+    @echo -n ${LIB_SOURCES} > rocksdb_lib_sources.txt
+
+gen_lib_sources: rocksdb_lib_sources.txt

rocksdb-sys/README.md

@@ -0,0 +1,10 @@
+RocksDB bindings
+================
+
+Low-level bindings to RocksDB's C API.
+
+Based on original work by Tyler Neely
+https://github.com/spacejam/rust-rocksdb
+and Jeremy Fitzhardinge
+https://github.com/jsgf/rocksdb-sys
+

rocksdb-sys/build.rs

@@ -0,0 +1,116 @@
+extern crate gcc;
+
+fn link(name: &str, bundled: bool) {
+    use std::env::var;
+    let target = var("TARGET").unwrap();
+    let target: Vec<_> = target.split('-').collect();
+    if target.get(2) == Some(&"windows") {
+        println!("cargo:rustc-link-lib=dylib={}", name);
+        if bundled && target.get(3) == Some(&"gnu") {
+            let dir = var("CARGO_MANIFEST_DIR").unwrap();
+            println!("cargo:rustc-link-search=native={}/{}", dir, target[0]);
+        }
+    }
+}
+
+fn build_rocksdb() {
+    let mut config = gcc::Config::new();
+    config.include("rocksdb/include/");
+    config.include("rocksdb/");
+    config.include("rocksdb/third-party/gtest-1.7.0/fused-src/");
+    config.include("snappy/");
+    config.include(".");
+
+    config.define("NDEBUG", Some("1"));
+    config.define("SNAPPY", Some("1"));
+
+    let mut lib_sources = include_str!("rocksdb_lib_sources.txt").split(" ").collect::<Vec<&'static str>>();
+
+    // We have a pregenerated a version of build_version.cc in the local directory
+    lib_sources = lib_sources.iter().cloned().filter(|file| {
+            *file != "util/build_version.cc"
+        })
+        .collect::<Vec<&'static str>>();
+
+    if cfg!(target_os = "macos") {
+        config.define("OS_MACOSX", Some("1"));
+        config.define("ROCKSDB_PLATFORM_POSIX", Some("1"));
+        config.define("ROCKSDB_LIB_IO_POSIX", Some("1"));
+
+    }
+    if cfg!(target_os = "linux") {
+        config.define("OS_LINUX", Some("1"));
+        config.define("ROCKSDB_PLATFORM_POSIX", Some("1"));
+        config.define("ROCKSDB_LIB_IO_POSIX", Some("1"));
+        //COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+    }
+    if cfg!(target_os = "freebsd") {
+        config.define("OS_FREEBSD", Some("1"));
+        config.define("ROCKSDB_PLATFORM_POSIX", Some("1"));
+        config.define("ROCKSDB_LIB_IO_POSIX", Some("1"));
+    }
+
+    if cfg!(windows) {
+        link("rpcrt4", false);
+        config.define("OS_WIN", Some("1"));
+
+        // Remove POSIX-specific sources
+        lib_sources = lib_sources.iter().cloned().filter(|file| {
+                match *file {
+                    "port/port_posix.cc" |
+                    "util/env_posix.cc" |
+                    "util/io_posix.cc" => false,
+                    _ => true
+                }
+            })
+            .collect::<Vec<&'static str>>();
+
+        // Add Windows-specific sources
+        lib_sources.push("port/win/port_win.cc");
+        lib_sources.push("port/win/env_win.cc");
+        lib_sources.push("port/win/env_default.cc");
+        lib_sources.push("port/win/win_logger.cc");
+        lib_sources.push("port/win/io_win.cc");
+    }
+
+    if cfg!(target_env = "msvc") {
+        config.flag("-EHsc");
+    } else {
+        config.flag("-std=c++11");
+    }
+
+    for file in lib_sources {
+        let file = "rocksdb/".to_string() + file;
+        config.file(&file);
+    }
+
+    config.file("build_version.cc");
+
+    config.cpp(true);
+    config.compile("librocksdb.a");
+}
+
+fn build_snappy() {
+    let mut config = gcc::Config::new();
+    config.include("snappy/");
+    config.include(".");
+
+    config.define("NDEBUG", Some("1"));
+
+    if cfg!(target_env = "msvc") {
+        config.flag("-EHsc");
+    } else {
+        config.flag("-std=c++11");
+    }
+
+    config.file("snappy/snappy.cc");
+    config.file("snappy/snappy-sinksource.cc");
+    config.file("snappy/snappy-c.cc");
+    config.cpp(true);
+    config.compile("libsnappy.a");
+}
+
+fn main() {
+    build_rocksdb();
+    build_snappy();
+}

rocksdb-sys/build_version.cc

@@ -0,0 +1,4 @@
+#include "build_version.h"
+const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:f201a44b4102308b840b15d9b89122af787476f1";
+const char* rocksdb_build_git_date = "rocksdb_build_git_date:2016-10-27";
+const char* rocksdb_build_compile_date = __DATE__;

rocksdb-sys/build_version.h

@@ -0,0 +1,15 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+#pragma once
+#if !defined(IOS_CROSS_COMPILE)
+// if we compile with Xcode, we don't run build_detect_vesion, so we don't
+// generate these variables
+// this variable tells us about the git revision
+extern const char* rocksdb_build_git_sha;
+
+// Date on which the code was compiled:
+extern const char* rocksdb_build_compile_date;
+#endif

rocksdb-sys/rocksdb_lib_sources.txt

@@ -0,0 +1 @@
+db/auto_roll_logger.cc db/builder.cc db/c.cc db/column_family.cc db/compacted_db_impl.cc db/compaction.cc db/compaction_iterator.cc db/compaction_job.cc db/compaction_picker.cc db/convenience.cc db/range_del_aggregator.cc db/db_filesnapshot.cc db/dbformat.cc db/db_impl.cc db/db_impl_debug.cc db/db_impl_readonly.cc db/db_impl_experimental.cc db/db_info_dumper.cc db/db_iter.cc db/external_sst_file_ingestion_job.cc db/experimental.cc db/event_helpers.cc db/file_indexer.cc db/filename.cc db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc db/internal_stats.cc db/log_reader.cc db/log_writer.cc db/managed_iterator.cc db/memtable_allocator.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc db/repair.cc db/snapshot_impl.cc db/table_cache.cc db/table_properties_collector.cc db/transaction_log_impl.cc db/version_builder.cc db/version_edit.cc db/version_set.cc db/wal_manager.cc db/write_batch.cc db/write_batch_base.cc db/write_controller.cc db/write_thread.cc db/xfunc_test_points.cc memtable/hash_cuckoo_rep.cc memtable/hash_linklist_rep.cc memtable/hash_skiplist_rep.cc memtable/skiplistrep.cc memtable/vectorrep.cc port/stack_trace.cc port/port_posix.cc table/adaptive_table_factory.cc table/block_based_filter_block.cc table/block_based_table_builder.cc table/block_based_table_factory.cc table/block_based_table_reader.cc table/block_builder.cc table/block.cc table/block_prefix_index.cc table/bloom_block.cc table/cuckoo_table_builder.cc table/cuckoo_table_factory.cc table/cuckoo_table_reader.cc table/flush_block_policy.cc table/format.cc table/full_filter_block.cc table/get_context.cc table/iterator.cc table/merger.cc table/meta_blocks.cc table/sst_file_writer.cc table/plain_table_builder.cc table/plain_table_factory.cc table/plain_table_index.cc table/plain_table_key_coding.cc table/plain_table_reader.cc table/persistent_cache_helper.cc table/table_properties.cc table/two_level_iterator.cc tools/dump/db_dump_tool.cc util/arena.cc util/bloom.cc util/build_version.cc util/cf_options.cc util/clock_cache.cc util/coding.cc util/comparator.cc util/compaction_job_stats_impl.cc util/concurrent_arena.cc util/crc32c.cc util/db_options.cc util/delete_scheduler.cc util/dynamic_bloom.cc util/env.cc util/env_chroot.cc util/env_hdfs.cc util/env_posix.cc util/event_logger.cc util/file_util.cc util/file_reader_writer.cc util/filter_policy.cc util/hash.cc util/histogram.cc util/histogram_windowing.cc util/instrumented_mutex.cc util/iostats_context.cc util/io_posix.cc util/log_buffer.cc util/logging.cc util/lru_cache.cc util/memenv.cc util/murmurhash.cc util/options.cc util/options_helper.cc util/options_parser.cc util/options_sanity_check.cc util/perf_context.cc util/perf_level.cc util/random.cc util/rate_limiter.cc util/sharded_cache.cc util/slice.cc util/sst_file_manager_impl.cc util/statistics.cc util/status.cc util/status_message.cc util/string_util.cc util/sync_point.cc util/thread_local.cc util/thread_status_impl.cc util/thread_status_updater.cc util/thread_status_updater_debug.cc util/thread_status_util.cc util/thread_status_util_debug.cc util/threadpool_imp.cc util/transaction_test_util.cc util/xfunc.cc util/xxhash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_db.cc utilities/convenience/info_log_finder.cc utilities/checkpoint/checkpoint.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/document/document_db.cc utilities/document/json_document_builder.cc utilities/document/json_document.cc utilities/env_mirror.cc utilities/env_registry.cc utilities/flashcache/flashcache.cc utilities/geodb/geodb_impl.cc utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc utilities/merge_operators/put.cc utilities/merge_operators/max.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/uint64add.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/block_cache_tier_file.cc utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/block_cache_tier.cc utilities/redis/redis_lists.cc utilities/simulator_cache/sim_cache.cc utilities/spatialdb/spatial_db.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/transactions/optimistic_transaction_impl.cc utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_impl.cc utilities/transactions/transaction_db_mutex_impl.cc utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_impl.cc utilities/transactions/transaction_util.cc utilities/ttl/db_ttl_impl.cc utilities/date_tiered/date_tiered_db_impl.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc

rocksdb-sys/snappy-stubs-public.h

@@ -0,0 +1,97 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+// Author: sesse@google.com (Steinar H. Gunderson)
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Various type stubs for the open-source version of Snappy.
+//
+// This file cannot include config.h, as it is included from snappy.h,
+// which is a public header. Instead, snappy-stubs-public.h is generated by
+// from snappy-stubs-public.h.in at configure time.
+
+#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
+#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
+
+#define HAVE_STDINT_H 1
+
+
+#include <stdint.h>
+
+#include <stddef.h>
+
+//#include <sys/uio.h>
+
+#if defined(_MSC_VER)
+#define ssize_t intptr_t
+#endif
+
+#define SNAPPY_MAJOR 1
+#define SNAPPY_MINOR 1
+#define SNAPPY_PATCHLEVEL 3
+#define SNAPPY_VERSION \
+    ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
+
+#include <string>
+
+namespace snappy {
+
+#if HAVE_STDINT_H
+typedef int8_t int8;
+typedef uint8_t uint8;
+typedef int16_t int16;
+typedef uint16_t uint16;
+typedef int32_t int32;
+typedef uint32_t uint32;
+typedef int64_t int64;
+typedef uint64_t uint64;
+#else
+typedef signed char int8;
+typedef unsigned char uint8;
+typedef short int16;
+typedef unsigned short uint16;
+typedef int int32;
+typedef unsigned int uint32;
+typedef long long int64;
+typedef unsigned long long uint64;
+#endif
+
+typedef std::string string;
+
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName&);               \
+  void operator=(const TypeName&)
+
+// Windows does not have an iovec type, yet the concept is universally useful.
+// It is simple to define it ourselves, so we put it inside our own namespace.
+struct iovec {
+    void* iov_base;
+    size_t iov_len;
+};
+
+}  // namespace snappy
+
+#endif  // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_

rocksdb-sys/src/lib.rs

@@ -0,0 +1,18 @@
+//
+// Copyright 2014 Tyler Neely
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+pub use ffi as rocksdb_ffi;
+pub use ffi::{DBCompactionStyle, DBComparator, DBCompressionType, DBRecoveryMode, new_bloom_filter};
+pub mod ffi;