Skip to content

Commit

Permalink
refactor: move serde_json::Value back to synth_core::Value
Browse files Browse the repository at this point in the history
  • Loading branch information
chesedo committed Jan 3, 2022
1 parent 7136617 commit 1800fea
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 180 deletions.
8 changes: 5 additions & 3 deletions synth/src/cli/import_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use async_std::task;
use log::debug;
use serde_json::Value;
use std::convert::TryFrom;
use synth_core::graph::json::synth_val_to_json;
use synth_core::schema::content::number_content::U64;
use synth_core::schema::{
ArrayContent, FieldRef, NumberContent, ObjectContent, OptionalMergeStrategy, RangeStep,
Expand Down Expand Up @@ -128,9 +129,10 @@ fn populate_namespace_values<T: DataSource + RelationalDataSource>(
) -> Result<()> {
task::block_on(datasource.set_seed())?;

for table in table_names {
let json_values = task::block_on(datasource.get_deterministic_samples(table))?;
namespace.try_update(OptionalMergeStrategy, table, &Value::from(json_values))?;
for table_name in table_names {
let values = task::block_on(datasource.get_deterministic_samples(table_name))?;
let json_values: Vec<Value> = values.into_iter().map(synth_val_to_json).collect();
namespace.try_update(OptionalMergeStrategy, table_name, &Value::from(json_values))?;
}

Ok(())
Expand Down
52 changes: 20 additions & 32 deletions synth/src/datasource/mysql_datasource.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ use rust_decimal::prelude::ToPrimitive;
use rust_decimal::Decimal;
use sqlx::mysql::{MySqlColumn, MySqlPoolOptions, MySqlQueryResult, MySqlRow};
use sqlx::{Column, MySql, Pool, Row, TypeInfo};
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::prelude::rust_2015::Result::Ok;
use synth_core::schema::number_content::{F64, I64, U64};
use synth_core::schema::{
ChronoValueType, DateTimeContent, NumberContent, RangeStep, RegexContent, StringContent,
};
use synth_core::{Content, Value};
use synth_gen::prelude::*;

/// TODO
/// Known issues:
Expand Down Expand Up @@ -128,7 +130,7 @@ impl RelationalDataSource for MySqlDataSource {
Ok(())
}

async fn get_deterministic_samples(&self, table_name: &str) -> Result<Vec<serde_json::Value>> {
async fn get_deterministic_samples(&self, table_name: &str) -> Result<Vec<Value>> {
let query = format!("SELECT * FROM {} ORDER BY rand(0.5) LIMIT 10", table_name);

sqlx::query(&query)
Expand Down Expand Up @@ -271,64 +273,50 @@ impl TryFrom<MySqlRow> for ValueWrapper {
type Error = anyhow::Error;

fn try_from(row: MySqlRow) -> Result<Self, Self::Error> {
let mut kv = serde_json::Map::new();
let mut kv = BTreeMap::new();

for column in row.columns() {
let value = try_match_value(&row, column).unwrap_or(serde_json::Value::Null);
let value = try_match_value(&row, column).unwrap_or(Value::Null(()));
kv.insert(column.name().to_string(), value);
}

Ok(ValueWrapper(serde_json::Value::Object(kv)))
Ok(ValueWrapper(Value::Object(kv)))
}
}

fn try_match_value(row: &MySqlRow, column: &MySqlColumn) -> Result<serde_json::Value> {
fn try_match_value(row: &MySqlRow, column: &MySqlColumn) -> Result<Value> {
let value = match column.type_info().name().to_lowercase().as_str() {
"char" | "varchar" | "text" | "binary" | "varbinary" | "enum" | "set" => {
serde_json::Value::String(row.try_get::<String, &str>(column.name())?)
Value::String(row.try_get::<String, &str>(column.name())?)
}
"tinyint" => serde_json::Value::Number(row.try_get::<i8, &str>(column.name())?.into()),
"smallint" => serde_json::Value::Number(row.try_get::<i16, &str>(column.name())?.into()),
"tinyint" => Value::Number(Number::from(row.try_get::<i8, &str>(column.name())?)),
"smallint" => Value::Number(Number::from(row.try_get::<i16, &str>(column.name())?)),
"mediumint" | "int" | "integer" => {
serde_json::Value::Number(row.try_get::<i32, &str>(column.name())?.into())
}
"bigint" => serde_json::Value::Number(row.try_get::<i64, &str>(column.name())?.into()),
"serial" => serde_json::Value::Number(row.try_get::<u64, &str>(column.name())?.into()),
"float" => {
let f = row.try_get::<f32, &str>(column.name())?;
let serde_f = serde_json::Number::from_f64(f as f64)
.ok_or_else(|| anyhow!("Failed to convert float4 to number"))?;
serde_json::Value::Number(serde_f)
}
"double" => {
let f = row.try_get::<f64, &str>(column.name())?;
let serde_f = serde_json::Number::from_f64(f)
.ok_or_else(|| anyhow!("Failed to convert float4 to number"))?;
serde_json::Value::Number(serde_f)
Value::Number(Number::from(row.try_get::<i32, &str>(column.name())?))
}
"bigint" => Value::Number(Number::from(row.try_get::<i64, &str>(column.name())?)),
"serial" => Value::Number(Number::from(row.try_get::<u64, &str>(column.name())?)),
"float" => Value::Number(Number::from(row.try_get::<f32, &str>(column.name())? as f64)),
"double" => Value::Number(Number::from(row.try_get::<f64, &str>(column.name())?)),
"numeric" | "decimal" => {
let as_decimal = row.try_get::<Decimal, &str>(column.name())?;

if let Some(truncated) = as_decimal.to_f64() {
return Ok(serde_json::Value::Number(
serde_json::Number::from_f64(truncated).ok_or_else(|| {
anyhow!("Failed to convert {} to number", column.type_info().name())
})?,
));
return Ok(Value::Number(Number::from(truncated)));
}

bail!("Failed to convert Mysql numeric data type to 64 bit float")
}
"timestamp" => serde_json::Value::String(row.try_get::<String, &str>(column.name())?),
"date" => serde_json::Value::String(format!(
"timestamp" => Value::String(row.try_get::<String, &str>(column.name())?),
"date" => Value::String(format!(
"{}",
row.try_get::<chrono::NaiveDate, &str>(column.name())?
)),
"datetime" => serde_json::Value::String(format!(
"datetime" => Value::String(format!(
"{}",
row.try_get::<chrono::NaiveDateTime, &str>(column.name())?
)),
"time" => serde_json::Value::String(format!(
"time" => Value::String(format!(
"{}",
row.try_get::<chrono::NaiveTime, &str>(column.name())?
)),
Expand Down
Loading

0 comments on commit 1800fea

Please sign in to comment.