@@ -8,8 +8,8 @@ edition = "2018"

[dependencies]
libtdb = {path = "../libtdb"}
tokio = {version = "0.3.6", features = ["full"]}
bytes = "0.6.0"
regex = "1.4.2"
tokio = {version = "1.2.0", features = ["full"]}
bytes = "1.0.1"
regex = "1.4.3"
lazy_static = "1.4.0"
clap = {version = "2.33.3", features=["yaml"]}
@@ -8,5 +8,5 @@ edition = "2018"

[dependencies]
lazy_static = "1.4.0"
bytes = "0.6.0"
termcolor = "1.1.1"
bytes = "1.0.1"
termcolor = "1.1.2"
@@ -7,24 +7,24 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
tokio = { version = "0.3.6", features = ["full"] }
bytes = "0.6.0"
tokio = { version = "1.2.0", features = ["full"] }
bytes = "1.0.1"
libtdb = {path ="../libtdb"}
bincode = "1.3.1"
parking_lot = "0.11.1"
lazy_static = "1.4.0"
serde_derive = "1.0.117"
serde = {version = "1.0.118", features= ["derive"]}
toml = "0.5.7"
serde_derive = "1.0.123"
serde = {version = "1.0.123", features= ["derive"]}
toml = "0.5.8"
clap = {version = "2.33.3", features=["yaml"]}
env_logger = "0.8.2"
log = "0.4.11"
env_logger = "0.8.3"
log = "0.4.14"
chrono = "0.4.19"
regex = "1.4.2"
regex = "1.4.3"
tdb_macros = {path="../tdb-macros"}

[target.'cfg(not(target_env = "msvc"))'.dependencies]
jemallocator = "0.3.2"

[dev-dependencies]
tokio = { version = "0.3.6", features = ["test-util"] }
tokio = { version = "1.2.0", features = ["test-util"] }
@@ -28,7 +28,7 @@ use crate::resp::GroupBegin;
use libtdb::terrapipe::RespCodes;
use libtdb::TResult;
use std::hint::unreachable_unchecked;
use std::path::PathBuf;
use std::path::{Component, PathBuf};

/// Create a snapshot
///
@@ -106,6 +106,25 @@ pub async fn mksnap(handle: &CoreDB, con: &mut Connection, act: ActionGroup) ->
let mut path = PathBuf::from(DIR_SNAPSHOT);
path.push("remote");
path.push(snapname.to_owned() + ".snapshot");
let illegal_snapshot = path
.components()
.filter(|dir| {
// Sanitize snapshot name, to avoid directory traversal attacks
// If the snapshot name has any root directory or parent directory, then
// we'll allow it to pass through this adaptor.
// As a result, this iterator will give us a count of the 'bad' components
dir == &Component::RootDir || dir == &Component::ParentDir
})
.count()
!= 0;
if illegal_snapshot {
con.write_response(GroupBegin(1)).await?;
return con
.write_response(RespCodes::OtherError(Some(
"err-invalid-snapshot-name".to_owned(),
)))
.await;
}
let failed;
{
match diskstore::flush_data(&path, &handle.acquire_read().get_ref()) {
@@ -131,4 +150,4 @@ pub async fn mksnap(handle: &CoreDB, con: &mut Connection, act: ActionGroup) ->
.await;
}
}
}
}
@@ -19,10 +19,10 @@
#
#

name: TerrabaseDB Server
name: Skybase Server
version: 0.5.0
author: Sayan N. <ohsayan@outlook.com>
about: The TerrabaseDB Database server
about: The Skybase Database server
args:
- config:
short: c
@@ -139,7 +139,7 @@ impl Listener {
loop {
// Take the permit first, but we won't use it right now
// that's why we will forget it
self.climit.acquire().await.forget();
self.climit.acquire().await.unwrap().forget();
let stream = self.accept().await?;
let mut chandle = CHandler {
db: self.db.clone(),
@@ -23,8 +23,7 @@
mod __private {
use crate::protocol::responses::fresp;
use libtdb::terrapipe;
use tokio::io::AsyncReadExt;
use tokio::prelude::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
/// Test a HEYA query: The server should return HEY!
async fn test_heya() {
let heya = terrapipe::proc_query("HEYA");
@@ -82,7 +81,6 @@ mod __private {
) where
T: AsRef<str>,
{
use tokio::prelude::*;
let mut query = String::from("MSET ");
query.push_str(values_split_with_whitespace.as_ref());
let count_bytes_len = homwany.to_string().as_bytes().len();
@@ -663,4 +661,21 @@ mod __private {
stream.read_exact(&mut response).await.unwrap();
assert_eq!(res_should_be, response);
}
async fn test_mksnap_sanitization() {
let res_should_be = "#2\n*1\n#2\n&1\n!25\nerr-invalid-snapshot-name\n"
.to_owned()
.into_bytes();
// First check parent directory syntax
let query = terrapipe::proc_query("MKSNAP ../../badsnappy");
stream.write_all(&query).await.unwrap();
let mut response = vec![0; res_should_be.len()];
stream.read_exact(&mut response).await.unwrap();
assert_eq!(res_should_be, response);
// Now check root directory syntax
let query = terrapipe::proc_query("MKSNAP /var/omgcrazysnappy");
stream.write_all(&query).await.unwrap();
let mut response = vec![0; res_should_be.len()];
stream.read_exact(&mut response).await.unwrap();
assert_eq!(res_should_be, response);
}
}
@@ -7,9 +7,9 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
rand = "0.7.3"
devtimer = "4.0.0"
rand = "0.8.3"
devtimer = "4.0.1"
libtdb = {path="../libtdb"}
clap = {version = "2.33.3", features=["yaml"]}
serde = {version="1.0.118", features=["derive"]}
serde_json = "1.0.60"
serde = {version="1.0.123", features=["derive"]}
serde_json = "1.0.62"
@@ -19,12 +19,12 @@
#
#

name: TerrabaseDB Benchmark Tool
version: 0.5.0
name: Skybase Benchmark Tool
version: 0.5-hotfix.1
author: Sayan N. <ohsayan@outlook.com>
about: |
The TerrabaseDB benchmark tool can be used to benchmark TerrabaseDB installations.
If you find any issues, then report one here: https://github.com/terrabasedb/terrabasedb
The Skybase benchmark tool can be used to benchmark Skybase installations.
If you find any issues, then report one here: https://github.com/skybasedb/skybase
args:
- connections:
short: c
@@ -1,7 +1,7 @@
/*
* Created on Sun Sep 13 2020
*
* This file is a part of TerrabaseDB
* This file is a part of Skybase
* Copyright (c) 2020, Sayan Nandan <ohsayan at outlook dot com>
*
* This program is free software: you can redistribute it and/or modify
@@ -28,7 +28,7 @@ mod benchtool {
use devtimer::DevTime;
use libtdb::terrapipe;
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use rand::thread_rng;
use serde::Serialize;
use std::io::prelude::*;
use std::net::{self, TcpStream};
@@ -167,25 +167,19 @@ mod benchtool {
},
None => host.push_str("2003"),
}
let rand = thread_rng();
let mut rand = thread_rng();
if let Some(matches) = matches.subcommand_matches("testkey") {
let numkeys = matches.value_of("count").unwrap();
if let Ok(num) = numkeys.parse::<usize>() {
let mut np = Netpool::new(10, &host);
println!("Generating keys ...");
let keys: Vec<String> = (0..num)
.into_iter()
.map(|_| {
let rand_string: String = rand.sample_iter(&Alphanumeric).take(8).collect();
rand_string
})
.map(|_| ran_string(8, &mut rand))
.collect();
let values: Vec<String> = (0..num)
.into_iter()
.map(|_| {
let rand_string: String = rand.sample_iter(&Alphanumeric).take(8).collect();
rand_string
})
.map(|_| ran_string(8, &mut rand))
.collect();
let set_packs: Vec<Vec<u8>> = (0..num)
.map(|idx| terrapipe::proc_query(format!("SET {} {}", keys[idx], values[idx])))
@@ -225,26 +219,18 @@ mod benchtool {
max_queries,
(packet_size * 2), // key size + value size
);
let rand = thread_rng();
let mut rand = thread_rng();
let mut dt = DevTime::new_complex();
// Create separate connection pools for get and set operations
let mut setpool = Netpool::new(max_connections, &host);
let mut getpool = Netpool::new(max_connections, &host);
let keys: Vec<String> = (0..max_queries)
.into_iter()
.map(|_| {
let rand_string: String =
rand.sample_iter(&Alphanumeric).take(packet_size).collect();
rand_string
})
.map(|_| ran_string(packet_size, &mut rand))
.collect();
let values: Vec<String> = (0..max_queries)
.into_iter()
.map(|_| {
let rand_string: String =
rand.sample_iter(&Alphanumeric).take(packet_size).collect();
rand_string
})
.map(|_| ran_string(packet_size, &mut rand))
.collect();
/*
We create three vectors of vectors: `set_packs`, `get_packs` and `del_packs`
@@ -309,8 +295,17 @@ mod benchtool {
fn calc(reqs: usize, time: u128) -> f64 {
reqs as f64 / (time as f64 / 1_000_000_000 as f64)
}

fn ran_string(len: usize, rand: impl rand::Rng) -> String {
let rand_string: String = rand
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)
.collect();
rand_string
}
}

fn main() {
benchtool::runner();
}
}
@@ -10,7 +10,7 @@ edition = "2018"
proc-macro = true

[dependencies]
syn = {version = "1.0.54", features = ["full"]}
quote = "1.0.7"
rand = "0.7.3"
syn = {version = "1.0.60", features = ["full"]}
quote = "1.0.9"
rand = "0.8.3"
proc-macro2 = "1.0.24"
@@ -67,7 +67,7 @@ fn parse_dbtest(mut input: syn::ItemFn, rand: u16) -> Result<TokenStream, syn::E
let addr = crate::tests::start_test_server(#rand, Some(asyncdb.clone())).await;
let mut stream = tokio::net::TcpStream::connect(&addr).await.unwrap();
#body
stream.shutdown(::std::net::Shutdown::Write).unwrap();
stream.shutdown().await.unwrap();
asyncdb.finish_db();
drop(asyncdb);
};
@@ -183,19 +183,19 @@ fn parse_test_module(args: TokenStream, item: TokenStream) -> TokenStream {
over by Hyper-V, which is why we'll prevent attempts to bind to them, if
the OS is Windows.
*/
let mut rand: u16 = rng.gen_range(1025, 65535);
let mut rand: u16 = rng.gen_range(1025..=65535);
#[cfg(not(target_os = "windows"))]
{
while in_set.contains(&rand) {
rand = rng.gen_range(1025, 65535);
rand = rng.gen_range(1025..=65535);
}
}
#[cfg(target_os = "windows")]
{
in_set.insert(5357);
in_set.insert(7680);
while in_set.contains(&rand) || (rand >= 49670 && rand <= 50293) {
rand = rng.gen_range(1025, 65535);
rand = rng.gen_range(1025..=65535);
}
}
in_set.insert(rand);