initial commit

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+*.iml
+
+/target
+**/*.rs.bk
+Cargo.lock

.travis.yml

@@ -0,0 +1,29 @@
+language: rust
+sudo: required
+dist: trusty
+addons:
+  apt:
+    packages:
+      - libssl-dev
+cache: cargo
+rust:
+  - stable
+  - beta
+  - nightly
+matrix:
+  allow_failures:
+    - rust: nightly
+script:
+  - cargo clean
+  - cargo build
+  - cargo test
+  - cargo doc --no-deps
+  - cargo package
+# cargo publish
+
+after_success: |
+  if [[ "$TRAVIS_RUST_VERSION" == stable ]]; then
+    cargo install cargo-tarpaulin &&
+    cargo tarpaulin --no-count --out Xml &&
+    bash <(curl -s https://codecov.io/bash)
+  fi

Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "pikmin"
+version = "0.1.0"
+authors = ["esplo <esplo@users.noreply.github.com>"]
+edition = "2018"
+
+description = "An extensible downloader for obtaining trade data simultaneously from exchanges' API."
+
+readme = "README.md"
+keywords = ["cryptocurrency", "downloader", "util"]
+categories = ["command-line-utilities"]
+license = "MIT"
+
+[dependencies]
+log = "^0.4"
+reqwest = "^0.9"
+serde = "^1.0"
+serde_derive = "^1.0"
+serde_json = "^1.0"
+chrono = { version = "^0.4", features = ["serde"] }
+mysql = { version = "^15.0", features = ["ssl"] }
+mysql_common = "^0.14.0"
+smallvec = "^0.6.8"
+
+[dev-dependencies]
+env_logger = "^0.6"
+tempfile = "3"
+
+[[example]]
+name = "main"
+
+[[example]]
+name = "simple"
+
+
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 esplo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,94 @@
+# Pikmin
+
+An extensible downloader for obtaining trade data simultaneously from exchanges' API.
+
+`pikmin` is a trade (execution) data downloader for crypto-currency exchanges,
+such as BitMex, bitFlyer, Liquid, etc. This library provides not only some pre-composed
+downloaders, but also ability to build a custom downloader for users' demand.
+
+## Pre-composed downloaders
+
+Currently, this library has the following downloaders:
+
+* BitMex (specify time, backwards to forwards)
+* bitFlyer (specify id, forwards to backwards)
+* Liquid (specify time, backwards to forwards)
+
+## Built-in Writer
+
+`Writer` is a processor between trade data and destination (typically DB).
+Pikmin has some built-in writers:
+
+* MySQL
+* stdout
+
+You can create your own writer easily.
+
+## Example
+
+A simple downloader for Liquid with writing to stdout.
+This program creates `./qn-progress.txt` for recording progress,
+so delete it if you want to run again from the starting point.
+
+```rust
+use std::path::Path;
+use std::thread;
+use std::time::Duration;
+
+use chrono::offset::TimeZone;
+use chrono::Utc;
+
+use pikmin::{LiquidDownloader, StdOutWriter};
+use pikmin::downloader::Downloader;
+
+fn main() {
+    // by using thread, you can run multiple downloaders
+    let liquid = thread::spawn(move || {
+        while {
+            // download data from 2019-01-01T01:01:01Z to 2019-01-01T01:03:01Z
+            // output the downloaded data to standard out (println!)
+            let downloader = LiquidDownloader::new(
+                Utc.ymd(2019, 1, 1).and_hms(1, 1, 1),
+                Utc.ymd(2019, 1, 1).and_hms(1, 3, 1),
+            );
+
+            // Locate progress file to `/tmp/qn-progress.txt`.
+            // You can control the starting point of downloading
+            // by preparing this file before you run.
+            let progress_file = Path::new("./qn-progress.txt");
+
+            // write out to standard out. simple writer for debugging
+            let mut writer = StdOutWriter::default();
+
+            println!("start QnDownloader");
+
+            // run!
+            match downloader.run(&mut writer, &progress_file) {
+                Ok(_) => {
+                    println!("finished");
+                    false
+                }
+                Err(e) => {
+                    println!("error: {}", e);
+                    println!("retry...");
+                    thread::sleep(Duration::from_secs(5));
+                    true
+                }
+            }
+        } {}
+    });
+
+    match liquid.join() {
+        Ok(_) => println!("finish all"),
+        Err(_) => println!("threading error"),
+    }
+}
+```
+
+Other examples can be found in `./examples`.
+
+## Future work
+
+- create pre-composed downloaders for other exchanges
+- parameterize the direction of downloading
+- abstraction of the progress writer

examples/main.rs

@@ -0,0 +1,128 @@
+use std::path::Path;
+use std::thread;
+use std::time::Duration;
+
+use chrono::offset::TimeZone;
+use chrono::Utc;
+
+use pikmin::{MySQLWriter, StdOutWriter};
+use pikmin::{BfDownloader, LiquidDownloader, MexDownloader};
+use pikmin::downloader::Downloader;
+
+// You can easily create a custom writer (this will be used in the `mex` example)
+struct CustomWriter {}
+
+impl pikmin::writer::Writer for CustomWriter {
+    fn write(&mut self, trades: &[pikmin::writer::Trade]) -> pikmin::error::Result<u64> {
+        for t in trades.iter() {
+            println!("{}", t.id);
+        }
+        Ok(trades.len() as u64)
+    }
+}
+
+fn main() {
+    // by using thread, you can run multiple downloaders
+    let liquid = thread::spawn(move || {
+        while {
+            // download data from 2019-01-01T01:01:01Z to 2019-01-01T01:03:01Z
+            // output the downloaded data to standard out (println!)
+            let downloader = LiquidDownloader::new(
+                Utc.ymd(2019, 1, 1).and_hms(1, 1, 1),
+                Utc.ymd(2019, 1, 1).and_hms(1, 3, 1),
+            );
+
+            // Locate progress file to `/tmp/qn-progress.txt`.
+            // You can control the starting point of downloading
+            // by preparing this file before you run.
+            let progress_file = Path::new("/tmp/qn-progress.txt");
+
+            // write out to standard out. simple writer for debugging
+            let mut writer = StdOutWriter::default();
+
+            println!("start QnDownloader");
+
+            // run!
+            match downloader.run(&mut writer, &progress_file) {
+                Ok(_) => {
+                    println!("finished");
+                    false
+                }
+                Err(e) => {
+                    println!("error: {}", e);
+                    println!("retry...");
+                    thread::sleep(Duration::from_secs(5));
+                    true
+                }
+            }
+        } {}
+    });
+
+    let bf = thread::spawn(move || {
+        while {
+            // Sadly, BitFlyer does not provide a pagination by a timestamp.
+            // Use u64::MAX and 0 to fetch all the data.
+            // Notice: execution data older than 1 month are not allowed to download on BitFlyer
+            let downloader = BfDownloader::new(764637994, 764677430);
+
+            let progress_file = Path::new("/tmp/bf-progress.txt");
+
+            // Make MySQL connection
+            let mut mysql_writer =
+                MySQLWriter::new("bf", "mysql://root:hoge@localhost:3333/trades");
+
+            println!("start BfDownloader");
+
+            match downloader.run(&mut mysql_writer, &progress_file) {
+                Ok(_) => {
+                    println!("finished");
+                    false
+                }
+                Err(e) => {
+                    println!("error: {}", e);
+                    println!("retry...");
+                    thread::sleep(Duration::from_secs(5));
+                    true
+                }
+            }
+        } {}
+    });
+
+    let mex = thread::spawn(move || {
+        while {
+            let downloader = MexDownloader::new(
+                Utc.ymd(2019, 1, 1).and_hms(1, 1, 1),
+                Utc.ymd(2019, 1, 1).and_hms(1, 3, 1),
+            );
+
+            let progress_file = Path::new("/tmp/mex-progress.txt");
+
+            // custom writer
+            let mut writer = CustomWriter {};
+
+            println!("start MexDownloader");
+
+            match downloader.run(&mut writer, &progress_file) {
+                Ok(_) => {
+                    println!("finished");
+                    false
+                }
+                Err(e) => {
+                    println!("error: {}", e);
+                    println!("retry...");
+                    thread::sleep(Duration::from_secs(5));
+                    true
+                }
+            }
+        } {}
+    });
+
+    match liquid
+        .join()
+        .and_then(|_| bf.join())
+        .and_then(|_| mex.join())
+        {
+            Ok(_) => println!("finish all"),
+            Err(_) => println!("threading error"),
+        }
+}

examples/simple.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+use std::thread;
+use std::time::Duration;
+
+use chrono::offset::TimeZone;
+use chrono::Utc;
+
+use pikmin::{LiquidDownloader, StdOutWriter};
+use pikmin::downloader::Downloader;
+
+fn main() {
+    // by using thread, you can run multiple downloaders
+    let liquid = thread::spawn(move || {
+        while {
+            // download data from 2019-01-01T01:01:01Z to 2019-01-01T01:03:01Z
+            // output the downloaded data to standard out (println!)
+            let downloader = LiquidDownloader::new(
+                Utc.ymd(2019, 1, 1).and_hms(1, 1, 1),
+                Utc.ymd(2019, 1, 1).and_hms(1, 3, 1),
+            );
+
+            // Locate progress file to `/tmp/qn-progress.txt`.
+            // You can control the starting point of downloading
+            // by preparing this file before you run.
+            let progress_file = Path::new("/tmp/qn-progress.txt");
+
+            // write out to standard out. simple writer for debugging
+            let mut writer = StdOutWriter::default();
+
+            println!("start QnDownloader");
+
+            // run!
+            match downloader.run(&mut writer, &progress_file) {
+                Ok(_) => {
+                    println!("finished");
+                    false
+                }
+                Err(e) => {
+                    println!("error: {}", e);
+                    println!("retry...");
+                    thread::sleep(Duration::from_secs(5));
+                    true
+                }
+            }
+        } {}
+    });
+
+    match liquid.join() {
+        Ok(_) => println!("finish all"),
+        Err(_) => println!("threading error"),
+    }
+}